{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Off-Policy VPG with Baseline to Play CartPole-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import losses\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:32:33 [INFO] env: <CartPoleEnv<CartPole-v0>>\n",
      "22:32:33 [INFO] action_space: Discrete(2)\n",
      "22:32:33 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)\n",
      "22:32:33 [INFO] reward_range: (-inf, inf)\n",
      "22:32:33 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}\n",
      "22:32:33 [INFO] _max_episode_steps: 200\n",
      "22:32:33 [INFO] _elapsed_steps: None\n",
      "22:32:33 [INFO] id: CartPole-v0\n",
      "22:32:33 [INFO] entry_point: gym.envs.classic_control:CartPoleEnv\n",
      "22:32:33 [INFO] reward_threshold: 195.0\n",
      "22:32:33 [INFO] nondeterministic: False\n",
      "22:32:33 [INFO] max_episode_steps: 200\n",
      "22:32:33 [INFO] _kwargs: {}\n",
      "22:32:33 [INFO] _env_name: CartPole\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OffPolicyVPGwBaselineAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        def dot(y_true, y_pred):\n",
    "            return -tf.reduce_sum(y_true * y_pred, axis=-1)\n",
    "\n",
    "        self.policy_net = self.build_net(hidden_sizes=[],\n",
    "                output_size=self.action_n,\n",
    "                output_activation=nn.softmax,\n",
    "                loss=dot, learning_rate=0.06)\n",
    "        self.baseline_net = self.build_net(hidden_sizes=[],\n",
    "                learning_rate=0.1)\n",
    "\n",
    "    def build_net(self, hidden_sizes, output_size=1,\n",
    "            activation=nn.relu, output_activation=None,\n",
    "            use_bias=False, loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation, use_bias=use_bias))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation, use_bias=use_bias))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train':\n",
    "            action = np.random.choice(self.action_n) # use random policy\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "        else:\n",
    "            probs = self.policy_net.predict(observation[np.newaxis])[0]\n",
    "            action = np.random.choice(self.action_n, p=probs)\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            self.learn()\n",
    "\n",
    "    def learn(self):\n",
    "        df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),\n",
    "                columns=['state', 'reward', 'done', 'action'])\n",
    "\n",
    "        # train baseline\n",
    "        df['discount'] = self.gamma ** df.index.to_series()\n",
    "        df['discounted_reward'] = df['discount'] * df['reward'].astype(float)\n",
    "        df['discounted_return'] = df['discounted_reward'][::-1].cumsum()\n",
    "        df['return'] = df['discounted_return'] / df['discount']\n",
    "        states = np.stack(df['state'])\n",
    "        returns = df[['return',]].values\n",
    "        self.baseline_net.fit(states, returns, verbose=0)\n",
    "\n",
    "        # train policy\n",
    "        states = np.stack(df['state'])\n",
    "        df['baseline'] = self.baseline_net.predict(states)\n",
    "        df['psi'] = df['discounted_return'] - df['baseline'] * df['discount']\n",
    "        df['behavior_prob'] = 1. / self.action_n\n",
    "        df['sample_weight'] = df['psi'] / df['behavior_prob']\n",
    "        actions = np.eye(self.action_n)[df['action'].astype(int)]\n",
    "        sample_weight = df[['sample_weight',]].values\n",
    "        self.policy_net.fit(states, actions, sample_weight=sample_weight, verbose=0)\n",
    "\n",
    "\n",
    "agent = OffPolicyVPGwBaselineAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:32:35 [INFO] ==== train ====\n",
      "22:32:35 [INFO] NumExpr defaulting to 8 threads.\n",
      "22:32:58 [DEBUG] train episode 0: reward = 119.00, steps = 119\n",
      "22:33:04 [DEBUG] train episode 1: reward = 31.00, steps = 31\n",
      "22:33:13 [DEBUG] train episode 2: reward = 60.00, steps = 60\n",
      "22:33:20 [DEBUG] train episode 3: reward = 40.00, steps = 40\n",
      "22:33:27 [DEBUG] train episode 4: reward = 42.00, steps = 42\n",
      "22:33:30 [DEBUG] train episode 5: reward = 15.00, steps = 15\n",
      "22:33:35 [DEBUG] train episode 6: reward = 22.00, steps = 22\n",
      "22:33:42 [DEBUG] train episode 7: reward = 38.00, steps = 38\n",
      "22:33:50 [DEBUG] train episode 8: reward = 44.00, steps = 44\n",
      "22:34:00 [DEBUG] train episode 9: reward = 62.00, steps = 62\n",
      "22:34:19 [DEBUG] train episode 10: reward = 102.00, steps = 102\n",
      "22:34:22 [DEBUG] train episode 11: reward = 16.00, steps = 16\n",
      "22:34:41 [DEBUG] train episode 12: reward = 134.00, steps = 134\n",
      "22:34:51 [DEBUG] train episode 13: reward = 59.00, steps = 59\n",
      "22:35:07 [DEBUG] train episode 14: reward = 111.00, steps = 111\n",
      "22:35:09 [DEBUG] train episode 15: reward = 16.00, steps = 16\n",
      "22:35:13 [DEBUG] train episode 16: reward = 25.00, steps = 25\n",
      "22:35:26 [DEBUG] train episode 17: reward = 88.00, steps = 88\n",
      "22:35:36 [DEBUG] train episode 18: reward = 79.00, steps = 79\n",
      "22:35:46 [DEBUG] train episode 19: reward = 59.00, steps = 59\n",
      "22:35:54 [DEBUG] train episode 20: reward = 61.00, steps = 61\n",
      "22:36:06 [DEBUG] train episode 21: reward = 82.00, steps = 82\n",
      "22:36:11 [DEBUG] train episode 22: reward = 33.00, steps = 33\n",
      "22:36:22 [DEBUG] train episode 23: reward = 76.00, steps = 76\n",
      "22:36:27 [DEBUG] train episode 24: reward = 32.00, steps = 32\n",
      "22:36:35 [DEBUG] train episode 25: reward = 58.00, steps = 58\n",
      "22:36:44 [DEBUG] train episode 26: reward = 73.00, steps = 73\n",
      "22:36:54 [DEBUG] train episode 27: reward = 75.00, steps = 75\n",
      "22:36:59 [DEBUG] train episode 28: reward = 38.00, steps = 38\n",
      "22:37:05 [DEBUG] train episode 29: reward = 50.00, steps = 50\n",
      "22:37:15 [DEBUG] train episode 30: reward = 70.00, steps = 70\n",
      "22:37:20 [DEBUG] train episode 31: reward = 38.00, steps = 38\n",
      "22:37:30 [DEBUG] train episode 32: reward = 71.00, steps = 71\n",
      "22:37:40 [DEBUG] train episode 33: reward = 63.00, steps = 63\n",
      "22:37:45 [DEBUG] train episode 34: reward = 29.00, steps = 29\n",
      "22:37:53 [DEBUG] train episode 35: reward = 41.00, steps = 41\n",
      "22:38:05 [DEBUG] train episode 36: reward = 69.00, steps = 69\n",
      "22:38:21 [DEBUG] train episode 37: reward = 96.00, steps = 96\n",
      "22:38:36 [DEBUG] train episode 38: reward = 93.00, steps = 93\n",
      "22:38:49 [DEBUG] train episode 39: reward = 100.00, steps = 100\n",
      "22:39:00 [DEBUG] train episode 40: reward = 76.00, steps = 76\n",
      "22:39:08 [DEBUG] train episode 41: reward = 47.00, steps = 47\n",
      "22:39:17 [DEBUG] train episode 42: reward = 58.00, steps = 58\n",
      "22:39:21 [DEBUG] train episode 43: reward = 31.00, steps = 31\n",
      "22:39:29 [DEBUG] train episode 44: reward = 57.00, steps = 57\n",
      "22:39:35 [DEBUG] train episode 45: reward = 41.00, steps = 41\n",
      "22:39:49 [DEBUG] train episode 46: reward = 100.00, steps = 100\n",
      "22:39:59 [DEBUG] train episode 47: reward = 74.00, steps = 74\n",
      "22:40:05 [DEBUG] train episode 48: reward = 43.00, steps = 43\n",
      "22:40:17 [DEBUG] train episode 49: reward = 86.00, steps = 86\n",
      "22:40:24 [DEBUG] train episode 50: reward = 49.00, steps = 49\n",
      "22:40:30 [DEBUG] train episode 51: reward = 40.00, steps = 40\n",
      "22:40:39 [DEBUG] train episode 52: reward = 69.00, steps = 69\n",
      "22:40:50 [DEBUG] train episode 53: reward = 73.00, steps = 73\n",
      "22:40:56 [DEBUG] train episode 54: reward = 43.00, steps = 43\n",
      "22:41:07 [DEBUG] train episode 55: reward = 75.00, steps = 75\n",
      "22:41:13 [DEBUG] train episode 56: reward = 46.00, steps = 46\n",
      "22:41:22 [DEBUG] train episode 57: reward = 57.00, steps = 57\n",
      "22:41:27 [DEBUG] train episode 58: reward = 38.00, steps = 38\n",
      "22:41:34 [DEBUG] train episode 59: reward = 54.00, steps = 54\n",
      "22:41:42 [DEBUG] train episode 60: reward = 54.00, steps = 54\n",
      "22:41:50 [DEBUG] train episode 61: reward = 61.00, steps = 61\n",
      "22:41:58 [DEBUG] train episode 62: reward = 61.00, steps = 61\n",
      "22:42:03 [DEBUG] train episode 63: reward = 30.00, steps = 30\n",
      "22:42:10 [DEBUG] train episode 64: reward = 65.00, steps = 65\n",
      "22:42:19 [DEBUG] train episode 65: reward = 69.00, steps = 69\n",
      "22:42:32 [DEBUG] train episode 66: reward = 104.00, steps = 104\n",
      "22:42:37 [DEBUG] train episode 67: reward = 41.00, steps = 41\n",
      "22:42:52 [DEBUG] train episode 68: reward = 114.00, steps = 114\n",
      "22:42:59 [DEBUG] train episode 69: reward = 56.00, steps = 56\n",
      "22:43:03 [DEBUG] train episode 70: reward = 34.00, steps = 34\n",
      "22:43:12 [DEBUG] train episode 71: reward = 68.00, steps = 68\n",
      "22:43:21 [DEBUG] train episode 72: reward = 74.00, steps = 74\n",
      "22:43:26 [DEBUG] train episode 73: reward = 39.00, steps = 39\n",
      "22:43:36 [DEBUG] train episode 74: reward = 80.00, steps = 80\n",
      "22:43:45 [DEBUG] train episode 75: reward = 68.00, steps = 68\n",
      "22:43:55 [DEBUG] train episode 76: reward = 83.00, steps = 83\n",
      "22:44:06 [DEBUG] train episode 77: reward = 87.00, steps = 87\n",
      "22:44:16 [DEBUG] train episode 78: reward = 78.00, steps = 78\n",
      "22:44:21 [DEBUG] train episode 79: reward = 39.00, steps = 39\n",
      "22:44:28 [DEBUG] train episode 80: reward = 54.00, steps = 54\n",
      "22:44:35 [DEBUG] train episode 81: reward = 56.00, steps = 56\n",
      "22:44:42 [DEBUG] train episode 82: reward = 50.00, steps = 50\n",
      "22:44:54 [DEBUG] train episode 83: reward = 93.00, steps = 93\n",
      "22:45:01 [DEBUG] train episode 84: reward = 55.00, steps = 55\n",
      "22:45:08 [DEBUG] train episode 85: reward = 54.00, steps = 54\n",
      "22:45:13 [DEBUG] train episode 86: reward = 41.00, steps = 41\n",
      "22:45:21 [DEBUG] train episode 87: reward = 61.00, steps = 61\n",
      "22:45:32 [DEBUG] train episode 88: reward = 88.00, steps = 88\n",
      "22:45:43 [DEBUG] train episode 89: reward = 84.00, steps = 84\n",
      "22:45:50 [DEBUG] train episode 90: reward = 51.00, steps = 51\n",
      "22:45:58 [DEBUG] train episode 91: reward = 61.00, steps = 61\n",
      "22:46:05 [DEBUG] train episode 92: reward = 55.00, steps = 55\n",
      "22:46:14 [DEBUG] train episode 93: reward = 65.00, steps = 65\n",
      "22:46:22 [DEBUG] train episode 94: reward = 56.00, steps = 56\n",
      "22:46:35 [DEBUG] train episode 95: reward = 86.00, steps = 86\n",
      "22:46:41 [DEBUG] train episode 96: reward = 34.00, steps = 34\n",
      "22:46:48 [DEBUG] train episode 97: reward = 50.00, steps = 50\n",
      "22:47:00 [DEBUG] train episode 98: reward = 92.00, steps = 92\n",
      "22:47:08 [DEBUG] train episode 99: reward = 63.00, steps = 63\n",
      "22:47:21 [DEBUG] train episode 100: reward = 97.00, steps = 97\n",
      "22:47:31 [DEBUG] train episode 101: reward = 71.00, steps = 71\n",
      "22:47:41 [DEBUG] train episode 102: reward = 64.00, steps = 64\n",
      "22:47:53 [DEBUG] train episode 103: reward = 84.00, steps = 84\n",
      "22:48:00 [DEBUG] train episode 104: reward = 54.00, steps = 54\n",
      "22:48:06 [DEBUG] train episode 105: reward = 43.00, steps = 43\n",
      "22:48:24 [DEBUG] train episode 106: reward = 144.00, steps = 144\n",
      "22:48:33 [DEBUG] train episode 107: reward = 58.00, steps = 58\n",
      "22:48:44 [DEBUG] train episode 108: reward = 79.00, steps = 79\n",
      "22:48:51 [DEBUG] train episode 109: reward = 48.00, steps = 48\n",
      "22:48:59 [DEBUG] train episode 110: reward = 56.00, steps = 56\n",
      "22:49:11 [DEBUG] train episode 111: reward = 86.00, steps = 86\n",
      "22:49:23 [DEBUG] train episode 112: reward = 92.00, steps = 92\n",
      "22:49:33 [DEBUG] train episode 113: reward = 73.00, steps = 73\n",
      "22:49:50 [DEBUG] train episode 114: reward = 126.00, steps = 126\n",
      "22:49:59 [DEBUG] train episode 115: reward = 64.00, steps = 64\n",
      "22:50:10 [DEBUG] train episode 116: reward = 77.00, steps = 77\n",
      "22:50:25 [DEBUG] train episode 117: reward = 91.00, steps = 91\n",
      "22:50:34 [DEBUG] train episode 118: reward = 66.00, steps = 66\n",
      "22:50:49 [DEBUG] train episode 119: reward = 98.00, steps = 98\n",
      "22:50:59 [DEBUG] train episode 120: reward = 48.00, steps = 48\n",
      "22:51:11 [DEBUG] train episode 121: reward = 72.00, steps = 72\n",
      "22:51:21 [DEBUG] train episode 122: reward = 56.00, steps = 56\n",
      "22:51:34 [DEBUG] train episode 123: reward = 79.00, steps = 79\n",
      "22:51:44 [DEBUG] train episode 124: reward = 62.00, steps = 62\n",
      "22:51:54 [DEBUG] train episode 125: reward = 66.00, steps = 66\n",
      "22:52:07 [DEBUG] train episode 126: reward = 106.00, steps = 106\n",
      "22:52:17 [DEBUG] train episode 127: reward = 73.00, steps = 73\n",
      "22:52:25 [DEBUG] train episode 128: reward = 58.00, steps = 58\n",
      "22:52:32 [DEBUG] train episode 129: reward = 45.00, steps = 45\n",
      "22:52:38 [DEBUG] train episode 130: reward = 42.00, steps = 42\n",
      "22:52:51 [DEBUG] train episode 131: reward = 96.00, steps = 96\n",
      "22:53:00 [DEBUG] train episode 132: reward = 66.00, steps = 66\n",
      "22:53:07 [DEBUG] train episode 133: reward = 49.00, steps = 49\n",
      "22:53:22 [DEBUG] train episode 134: reward = 107.00, steps = 107\n",
      "22:53:28 [DEBUG] train episode 135: reward = 42.00, steps = 42\n",
      "22:53:36 [DEBUG] train episode 136: reward = 55.00, steps = 55\n",
      "22:53:48 [DEBUG] train episode 137: reward = 75.00, steps = 75\n",
      "22:54:01 [DEBUG] train episode 138: reward = 70.00, steps = 70\n",
      "22:54:13 [DEBUG] train episode 139: reward = 59.00, steps = 59\n",
      "22:54:20 [DEBUG] train episode 140: reward = 35.00, steps = 35\n",
      "22:54:41 [DEBUG] train episode 141: reward = 105.00, steps = 105\n",
      "22:54:53 [DEBUG] train episode 142: reward = 65.00, steps = 65\n",
      "22:55:02 [DEBUG] train episode 143: reward = 48.00, steps = 48\n",
      "22:55:14 [DEBUG] train episode 144: reward = 62.00, steps = 62\n",
      "22:55:37 [DEBUG] train episode 145: reward = 109.00, steps = 109\n",
      "22:55:49 [DEBUG] train episode 146: reward = 64.00, steps = 64\n",
      "22:56:09 [DEBUG] train episode 147: reward = 102.00, steps = 102\n",
      "22:56:33 [DEBUG] train episode 148: reward = 113.00, steps = 113\n",
      "22:56:46 [DEBUG] train episode 149: reward = 64.00, steps = 64\n",
      "22:56:58 [DEBUG] train episode 150: reward = 52.00, steps = 52\n",
      "22:57:13 [DEBUG] train episode 151: reward = 68.00, steps = 68\n",
      "22:57:24 [DEBUG] train episode 152: reward = 49.00, steps = 49\n",
      "22:57:41 [DEBUG] train episode 153: reward = 88.00, steps = 88\n",
      "22:57:51 [DEBUG] train episode 154: reward = 46.00, steps = 46\n",
      "22:58:06 [DEBUG] train episode 155: reward = 82.00, steps = 82\n",
      "22:58:29 [DEBUG] train episode 156: reward = 130.00, steps = 130\n",
      "22:58:41 [DEBUG] train episode 157: reward = 63.00, steps = 63\n",
      "22:58:56 [DEBUG] train episode 158: reward = 78.00, steps = 78\n",
      "22:59:05 [DEBUG] train episode 159: reward = 57.00, steps = 57\n",
      "22:59:17 [DEBUG] train episode 160: reward = 77.00, steps = 77\n",
      "22:59:30 [DEBUG] train episode 161: reward = 77.00, steps = 77\n",
      "22:59:38 [DEBUG] train episode 162: reward = 51.00, steps = 51\n",
      "22:59:58 [DEBUG] train episode 163: reward = 123.00, steps = 123\n",
      "23:00:24 [DEBUG] train episode 164: reward = 177.00, steps = 177\n",
      "23:00:38 [DEBUG] train episode 165: reward = 90.00, steps = 90\n",
      "23:00:46 [DEBUG] train episode 166: reward = 54.00, steps = 54\n",
      "23:00:55 [DEBUG] train episode 167: reward = 56.00, steps = 56\n",
      "23:01:02 [DEBUG] train episode 168: reward = 42.00, steps = 42\n",
      "23:01:13 [DEBUG] train episode 169: reward = 78.00, steps = 78\n",
      "23:01:21 [DEBUG] train episode 170: reward = 63.00, steps = 63\n",
      "23:01:32 [DEBUG] train episode 171: reward = 86.00, steps = 86\n",
      "23:01:41 [DEBUG] train episode 172: reward = 66.00, steps = 66\n",
      "23:01:48 [DEBUG] train episode 173: reward = 59.00, steps = 59\n",
      "23:01:56 [DEBUG] train episode 174: reward = 55.00, steps = 55\n",
      "23:02:04 [DEBUG] train episode 175: reward = 58.00, steps = 58\n",
      "23:02:14 [DEBUG] train episode 176: reward = 80.00, steps = 80\n",
      "23:02:30 [DEBUG] train episode 177: reward = 120.00, steps = 120\n",
      "23:02:38 [DEBUG] train episode 178: reward = 56.00, steps = 56\n",
      "23:02:46 [DEBUG] train episode 179: reward = 59.00, steps = 59\n",
      "23:03:03 [DEBUG] train episode 180: reward = 130.00, steps = 130\n",
      "23:03:14 [DEBUG] train episode 181: reward = 92.00, steps = 92\n",
      "23:03:22 [DEBUG] train episode 182: reward = 61.00, steps = 61\n",
      "23:03:31 [DEBUG] train episode 183: reward = 64.00, steps = 64\n",
      "23:03:40 [DEBUG] train episode 184: reward = 70.00, steps = 70\n",
      "23:03:50 [DEBUG] train episode 185: reward = 76.00, steps = 76\n",
      "23:03:56 [DEBUG] train episode 186: reward = 49.00, steps = 49\n",
      "23:04:07 [DEBUG] train episode 187: reward = 82.00, steps = 82\n",
      "23:04:21 [DEBUG] train episode 188: reward = 118.00, steps = 118\n",
      "23:04:30 [DEBUG] train episode 189: reward = 69.00, steps = 69\n",
      "23:04:41 [DEBUG] train episode 190: reward = 91.00, steps = 91\n",
      "23:04:48 [DEBUG] train episode 191: reward = 59.00, steps = 59\n",
      "23:04:59 [DEBUG] train episode 192: reward = 88.00, steps = 88\n",
      "23:05:10 [DEBUG] train episode 193: reward = 88.00, steps = 88\n",
      "23:05:23 [DEBUG] train episode 194: reward = 107.00, steps = 107\n",
      "23:05:32 [DEBUG] train episode 195: reward = 72.00, steps = 72\n",
      "23:05:41 [DEBUG] train episode 196: reward = 76.00, steps = 76\n",
      "23:05:51 [DEBUG] train episode 197: reward = 78.00, steps = 78\n",
      "23:05:58 [DEBUG] train episode 198: reward = 57.00, steps = 57\n",
      "23:06:05 [DEBUG] train episode 199: reward = 60.00, steps = 60\n",
      "23:06:11 [DEBUG] train episode 200: reward = 50.00, steps = 50\n",
      "23:06:19 [DEBUG] train episode 201: reward = 65.00, steps = 65\n",
      "23:06:24 [DEBUG] train episode 202: reward = 39.00, steps = 39\n",
      "23:06:33 [DEBUG] train episode 203: reward = 68.00, steps = 68\n",
      "23:06:44 [DEBUG] train episode 204: reward = 98.00, steps = 98\n",
      "23:06:53 [DEBUG] train episode 205: reward = 69.00, steps = 69\n",
      "23:07:11 [DEBUG] train episode 206: reward = 148.00, steps = 148\n",
      "23:07:18 [DEBUG] train episode 207: reward = 63.00, steps = 63\n",
      "23:07:28 [DEBUG] train episode 208: reward = 79.00, steps = 79\n",
      "23:07:33 [DEBUG] train episode 209: reward = 42.00, steps = 42\n",
      "23:07:40 [DEBUG] train episode 210: reward = 54.00, steps = 54\n",
      "23:07:47 [DEBUG] train episode 211: reward = 57.00, steps = 57\n",
      "23:08:04 [DEBUG] train episode 212: reward = 142.00, steps = 142\n",
      "23:08:11 [DEBUG] train episode 213: reward = 55.00, steps = 55\n",
      "23:08:24 [DEBUG] train episode 214: reward = 109.00, steps = 109\n",
      "23:08:38 [DEBUG] train episode 215: reward = 114.00, steps = 114\n",
      "23:08:45 [DEBUG] train episode 216: reward = 57.00, steps = 57\n",
      "23:08:58 [DEBUG] train episode 217: reward = 110.00, steps = 110\n",
      "23:09:07 [DEBUG] train episode 218: reward = 68.00, steps = 68\n",
      "23:09:17 [DEBUG] train episode 219: reward = 89.00, steps = 89\n",
      "23:09:28 [DEBUG] train episode 220: reward = 77.00, steps = 77\n",
      "23:09:37 [DEBUG] train episode 221: reward = 62.00, steps = 62\n",
      "23:09:46 [DEBUG] train episode 222: reward = 54.00, steps = 54\n",
      "23:09:53 [DEBUG] train episode 223: reward = 53.00, steps = 53\n",
      "23:10:03 [DEBUG] train episode 224: reward = 69.00, steps = 69\n",
      "23:10:13 [DEBUG] train episode 225: reward = 67.00, steps = 67\n",
      "23:10:22 [DEBUG] train episode 226: reward = 57.00, steps = 57\n",
      "23:10:36 [DEBUG] train episode 227: reward = 100.00, steps = 100\n",
      "23:10:47 [DEBUG] train episode 228: reward = 69.00, steps = 69\n",
      "23:11:00 [DEBUG] train episode 229: reward = 95.00, steps = 95\n",
      "23:11:12 [DEBUG] train episode 230: reward = 74.00, steps = 74\n",
      "23:11:34 [DEBUG] train episode 231: reward = 144.00, steps = 144\n",
      "23:11:43 [DEBUG] train episode 232: reward = 70.00, steps = 70\n",
      "23:11:53 [DEBUG] train episode 233: reward = 81.00, steps = 81\n",
      "23:12:04 [DEBUG] train episode 234: reward = 79.00, steps = 79\n",
      "23:12:16 [DEBUG] train episode 235: reward = 87.00, steps = 87\n",
      "23:12:30 [DEBUG] train episode 236: reward = 110.00, steps = 110\n",
      "23:12:35 [DEBUG] train episode 237: reward = 47.00, steps = 47\n",
      "23:12:45 [DEBUG] train episode 238: reward = 77.00, steps = 77\n",
      "23:12:54 [DEBUG] train episode 239: reward = 74.00, steps = 74\n",
      "23:13:07 [DEBUG] train episode 240: reward = 103.00, steps = 103\n",
      "23:13:15 [DEBUG] train episode 241: reward = 64.00, steps = 64\n",
      "23:13:24 [DEBUG] train episode 242: reward = 70.00, steps = 70\n",
      "23:13:31 [DEBUG] train episode 243: reward = 57.00, steps = 57\n",
      "23:13:43 [DEBUG] train episode 244: reward = 97.00, steps = 97\n",
      "23:13:54 [DEBUG] train episode 245: reward = 91.00, steps = 91\n",
      "23:14:07 [DEBUG] train episode 246: reward = 104.00, steps = 104\n",
      "23:14:21 [DEBUG] train episode 247: reward = 117.00, steps = 117\n",
      "23:14:28 [DEBUG] train episode 248: reward = 55.00, steps = 55\n",
      "23:14:36 [DEBUG] train episode 249: reward = 69.00, steps = 69\n",
      "23:14:52 [DEBUG] train episode 250: reward = 131.00, steps = 131\n",
      "23:15:06 [DEBUG] train episode 251: reward = 115.00, steps = 115\n",
      "23:15:14 [DEBUG] train episode 252: reward = 62.00, steps = 62\n",
      "23:15:24 [DEBUG] train episode 253: reward = 88.00, steps = 88\n",
      "23:15:37 [DEBUG] train episode 254: reward = 108.00, steps = 108\n",
      "23:15:52 [DEBUG] train episode 255: reward = 114.00, steps = 114\n",
      "23:16:01 [DEBUG] train episode 256: reward = 74.00, steps = 74\n",
      "23:16:14 [DEBUG] train episode 257: reward = 111.00, steps = 111\n",
      "23:16:28 [DEBUG] train episode 258: reward = 118.00, steps = 118\n",
      "23:16:44 [DEBUG] train episode 259: reward = 124.00, steps = 124\n",
      "23:16:50 [DEBUG] train episode 260: reward = 47.00, steps = 47\n",
      "23:17:04 [DEBUG] train episode 261: reward = 116.00, steps = 116\n",
      "23:17:12 [DEBUG] train episode 262: reward = 68.00, steps = 68\n",
      "23:17:19 [DEBUG] train episode 263: reward = 60.00, steps = 60\n",
      "23:17:32 [DEBUG] train episode 264: reward = 105.00, steps = 105\n",
      "23:17:39 [DEBUG] train episode 265: reward = 51.00, steps = 51\n",
      "23:17:44 [DEBUG] train episode 266: reward = 46.00, steps = 46\n",
      "23:17:57 [DEBUG] train episode 267: reward = 108.00, steps = 108\n",
      "23:18:05 [DEBUG] train episode 268: reward = 65.00, steps = 65\n",
      "23:18:11 [DEBUG] train episode 269: reward = 43.00, steps = 43\n",
      "23:18:23 [DEBUG] train episode 270: reward = 105.00, steps = 105\n",
      "23:18:31 [DEBUG] train episode 271: reward = 61.00, steps = 61\n",
      "23:18:41 [DEBUG] train episode 272: reward = 81.00, steps = 81\n",
      "23:18:50 [DEBUG] train episode 273: reward = 73.00, steps = 73\n",
      "23:19:06 [DEBUG] train episode 274: reward = 112.00, steps = 112\n",
      "23:19:18 [DEBUG] train episode 275: reward = 85.00, steps = 85\n",
      "23:19:30 [DEBUG] train episode 276: reward = 88.00, steps = 88\n",
      "23:19:41 [DEBUG] train episode 277: reward = 76.00, steps = 76\n",
      "23:19:50 [DEBUG] train episode 278: reward = 71.00, steps = 71\n",
      "23:19:58 [DEBUG] train episode 279: reward = 51.00, steps = 51\n",
      "23:20:05 [DEBUG] train episode 280: reward = 55.00, steps = 55\n",
      "23:20:14 [DEBUG] train episode 281: reward = 61.00, steps = 61\n",
      "23:20:21 [DEBUG] train episode 282: reward = 55.00, steps = 55\n",
      "23:20:31 [DEBUG] train episode 283: reward = 75.00, steps = 75\n",
      "23:20:38 [DEBUG] train episode 284: reward = 59.00, steps = 59\n",
      "23:20:46 [DEBUG] train episode 285: reward = 64.00, steps = 64\n",
      "23:20:54 [DEBUG] train episode 286: reward = 67.00, steps = 67\n",
      "23:21:04 [DEBUG] train episode 287: reward = 80.00, steps = 80\n",
      "23:21:13 [DEBUG] train episode 288: reward = 69.00, steps = 69\n",
      "23:21:30 [DEBUG] train episode 289: reward = 140.00, steps = 140\n",
      "23:21:42 [DEBUG] train episode 290: reward = 90.00, steps = 90\n",
      "23:21:55 [DEBUG] train episode 291: reward = 110.00, steps = 110\n",
      "23:22:10 [DEBUG] train episode 292: reward = 109.00, steps = 109\n",
      "23:22:20 [DEBUG] train episode 293: reward = 82.00, steps = 82\n",
      "23:22:29 [DEBUG] train episode 294: reward = 60.00, steps = 60\n",
      "23:22:40 [DEBUG] train episode 295: reward = 76.00, steps = 76\n",
      "23:22:56 [DEBUG] train episode 296: reward = 102.00, steps = 102\n",
      "23:23:04 [DEBUG] train episode 297: reward = 57.00, steps = 57\n",
      "23:23:12 [DEBUG] train episode 298: reward = 63.00, steps = 63\n",
      "23:23:24 [DEBUG] train episode 299: reward = 89.00, steps = 89\n",
      "23:23:32 [DEBUG] train episode 300: reward = 56.00, steps = 56\n",
      "23:23:41 [DEBUG] train episode 301: reward = 68.00, steps = 68\n",
      "23:23:52 [DEBUG] train episode 302: reward = 85.00, steps = 85\n",
      "23:24:03 [DEBUG] train episode 303: reward = 83.00, steps = 83\n",
      "23:24:14 [DEBUG] train episode 304: reward = 78.00, steps = 78\n",
      "23:24:25 [DEBUG] train episode 305: reward = 81.00, steps = 81\n",
      "23:24:32 [DEBUG] train episode 306: reward = 55.00, steps = 55\n",
      "23:24:45 [DEBUG] train episode 307: reward = 95.00, steps = 95\n",
      "23:24:53 [DEBUG] train episode 308: reward = 57.00, steps = 57\n",
      "23:25:03 [DEBUG] train episode 309: reward = 77.00, steps = 77\n",
      "23:25:14 [DEBUG] train episode 310: reward = 92.00, steps = 92\n",
      "23:25:24 [DEBUG] train episode 311: reward = 69.00, steps = 69\n",
      "23:25:45 [DEBUG] train episode 312: reward = 169.00, steps = 169\n",
      "23:25:52 [DEBUG] train episode 313: reward = 56.00, steps = 56\n",
      "23:26:04 [DEBUG] train episode 314: reward = 85.00, steps = 85\n",
      "23:26:14 [DEBUG] train episode 315: reward = 76.00, steps = 76\n",
      "23:26:25 [DEBUG] train episode 316: reward = 77.00, steps = 77\n",
      "23:26:47 [DEBUG] train episode 317: reward = 149.00, steps = 149\n",
      "23:27:04 [DEBUG] train episode 318: reward = 134.00, steps = 134\n",
      "23:27:15 [DEBUG] train episode 319: reward = 81.00, steps = 81\n",
      "23:27:26 [DEBUG] train episode 320: reward = 76.00, steps = 76\n",
      "23:27:33 [DEBUG] train episode 321: reward = 53.00, steps = 53\n",
      "23:27:43 [DEBUG] train episode 322: reward = 74.00, steps = 74\n",
      "23:28:07 [DEBUG] train episode 323: reward = 171.00, steps = 171\n",
      "23:28:18 [DEBUG] train episode 324: reward = 74.00, steps = 74\n",
      "23:28:25 [DEBUG] train episode 325: reward = 48.00, steps = 48\n",
      "23:28:32 [DEBUG] train episode 326: reward = 48.00, steps = 48\n",
      "23:28:42 [DEBUG] train episode 327: reward = 78.00, steps = 78\n",
      "23:28:57 [DEBUG] train episode 328: reward = 105.00, steps = 105\n",
      "23:29:08 [DEBUG] train episode 329: reward = 72.00, steps = 72\n",
      "23:29:16 [DEBUG] train episode 330: reward = 57.00, steps = 57\n",
      "23:29:42 [DEBUG] train episode 331: reward = 200.00, steps = 200\n",
      "23:29:55 [DEBUG] train episode 332: reward = 97.00, steps = 97\n",
      "23:30:05 [DEBUG] train episode 333: reward = 80.00, steps = 80\n",
      "23:30:18 [DEBUG] train episode 334: reward = 94.00, steps = 94\n",
      "23:30:29 [DEBUG] train episode 335: reward = 84.00, steps = 84\n",
      "23:30:40 [DEBUG] train episode 336: reward = 81.00, steps = 81\n",
      "23:30:51 [DEBUG] train episode 337: reward = 79.00, steps = 79\n",
      "23:30:59 [DEBUG] train episode 338: reward = 61.00, steps = 61\n",
      "23:31:10 [DEBUG] train episode 339: reward = 97.00, steps = 97\n",
      "23:31:29 [DEBUG] train episode 340: reward = 154.00, steps = 154\n",
      "23:31:37 [DEBUG] train episode 341: reward = 75.00, steps = 75\n",
      "23:31:50 [DEBUG] train episode 342: reward = 108.00, steps = 108\n",
      "23:32:02 [DEBUG] train episode 343: reward = 71.00, steps = 71\n",
      "23:32:18 [DEBUG] train episode 344: reward = 137.00, steps = 137\n",
      "23:32:26 [DEBUG] train episode 345: reward = 63.00, steps = 63\n",
      "23:32:39 [DEBUG] train episode 346: reward = 109.00, steps = 109\n",
      "23:32:49 [DEBUG] train episode 347: reward = 89.00, steps = 89\n",
      "23:33:00 [DEBUG] train episode 348: reward = 97.00, steps = 97\n",
      "23:33:15 [DEBUG] train episode 349: reward = 140.00, steps = 140\n",
      "23:33:24 [DEBUG] train episode 350: reward = 84.00, steps = 84\n",
      "23:33:35 [DEBUG] train episode 351: reward = 92.00, steps = 92\n",
      "23:33:45 [DEBUG] train episode 352: reward = 90.00, steps = 90\n",
      "23:33:55 [DEBUG] train episode 353: reward = 94.00, steps = 94\n",
      "23:34:03 [DEBUG] train episode 354: reward = 84.00, steps = 84\n",
      "23:34:18 [DEBUG] train episode 355: reward = 94.00, steps = 94\n",
      "23:34:24 [DEBUG] train episode 356: reward = 59.00, steps = 59\n",
      "23:34:31 [DEBUG] train episode 357: reward = 65.00, steps = 65\n",
      "23:34:48 [DEBUG] train episode 358: reward = 170.00, steps = 170\n",
      "23:34:56 [DEBUG] train episode 359: reward = 76.00, steps = 76\n",
      "23:35:05 [DEBUG] train episode 360: reward = 83.00, steps = 83\n",
      "23:35:15 [DEBUG] train episode 361: reward = 87.00, steps = 87\n",
      "23:35:23 [DEBUG] train episode 362: reward = 68.00, steps = 68\n",
      "23:35:31 [DEBUG] train episode 363: reward = 69.00, steps = 69\n",
      "23:35:37 [DEBUG] train episode 364: reward = 56.00, steps = 56\n",
      "23:35:49 [DEBUG] train episode 365: reward = 107.00, steps = 107\n",
      "23:36:03 [DEBUG] train episode 366: reward = 147.00, steps = 147\n",
      "23:36:13 [DEBUG] train episode 367: reward = 94.00, steps = 94\n",
      "23:36:23 [DEBUG] train episode 368: reward = 99.00, steps = 99\n",
      "23:36:29 [DEBUG] train episode 369: reward = 57.00, steps = 57\n",
      "23:36:40 [DEBUG] train episode 370: reward = 109.00, steps = 109\n",
      "23:36:49 [DEBUG] train episode 371: reward = 80.00, steps = 80\n",
      "23:36:58 [DEBUG] train episode 372: reward = 93.00, steps = 93\n",
      "23:37:06 [DEBUG] train episode 373: reward = 73.00, steps = 73\n",
      "23:37:12 [DEBUG] train episode 374: reward = 57.00, steps = 57\n",
      "23:37:19 [DEBUG] train episode 375: reward = 59.00, steps = 59\n",
      "23:37:28 [DEBUG] train episode 376: reward = 86.00, steps = 86\n",
      "23:37:35 [DEBUG] train episode 377: reward = 79.00, steps = 79\n",
      "23:37:43 [DEBUG] train episode 378: reward = 67.00, steps = 67\n",
      "23:37:57 [DEBUG] train episode 379: reward = 134.00, steps = 134\n",
      "23:38:06 [DEBUG] train episode 380: reward = 85.00, steps = 85\n",
      "23:38:14 [DEBUG] train episode 381: reward = 86.00, steps = 86\n",
      "23:38:24 [DEBUG] train episode 382: reward = 90.00, steps = 90\n",
      "23:38:33 [DEBUG] train episode 383: reward = 90.00, steps = 90\n",
      "23:38:46 [DEBUG] train episode 384: reward = 122.00, steps = 122\n",
      "23:38:54 [DEBUG] train episode 385: reward = 77.00, steps = 77\n",
      "23:39:04 [DEBUG] train episode 386: reward = 91.00, steps = 91\n",
      "23:39:12 [DEBUG] train episode 387: reward = 71.00, steps = 71\n",
      "23:39:25 [DEBUG] train episode 388: reward = 115.00, steps = 115\n",
      "23:39:40 [DEBUG] train episode 389: reward = 133.00, steps = 133\n",
      "23:39:48 [DEBUG] train episode 390: reward = 72.00, steps = 72\n",
      "23:39:57 [DEBUG] train episode 391: reward = 81.00, steps = 81\n",
      "23:40:05 [DEBUG] train episode 392: reward = 68.00, steps = 68\n",
      "23:40:21 [DEBUG] train episode 393: reward = 142.00, steps = 142\n",
      "23:40:38 [DEBUG] train episode 394: reward = 154.00, steps = 154\n",
      "23:40:46 [DEBUG] train episode 395: reward = 70.00, steps = 70\n",
      "23:41:01 [DEBUG] train episode 396: reward = 129.00, steps = 129\n",
      "23:41:19 [DEBUG] train episode 397: reward = 162.00, steps = 162\n",
      "23:41:33 [DEBUG] train episode 398: reward = 134.00, steps = 134\n",
      "23:41:47 [DEBUG] train episode 399: reward = 135.00, steps = 135\n",
      "23:41:56 [DEBUG] train episode 400: reward = 83.00, steps = 83\n",
      "23:42:11 [DEBUG] train episode 401: reward = 142.00, steps = 142\n",
      "23:42:18 [DEBUG] train episode 402: reward = 72.00, steps = 72\n",
      "23:42:27 [DEBUG] train episode 403: reward = 78.00, steps = 78\n",
      "23:42:41 [DEBUG] train episode 404: reward = 139.00, steps = 139\n",
      "23:42:51 [DEBUG] train episode 405: reward = 90.00, steps = 90\n",
      "23:42:59 [DEBUG] train episode 406: reward = 76.00, steps = 76\n",
      "23:43:09 [DEBUG] train episode 407: reward = 95.00, steps = 95\n",
      "23:43:16 [DEBUG] train episode 408: reward = 56.00, steps = 56\n",
      "23:43:26 [DEBUG] train episode 409: reward = 85.00, steps = 85\n",
      "23:43:42 [DEBUG] train episode 410: reward = 141.00, steps = 141\n",
      "23:43:54 [DEBUG] train episode 411: reward = 89.00, steps = 89\n",
      "23:44:02 [DEBUG] train episode 412: reward = 67.00, steps = 67\n",
      "23:44:09 [DEBUG] train episode 413: reward = 66.00, steps = 66\n",
      "23:44:24 [DEBUG] train episode 414: reward = 115.00, steps = 115\n",
      "23:44:34 [DEBUG] train episode 415: reward = 86.00, steps = 86\n",
      "23:44:42 [DEBUG] train episode 416: reward = 73.00, steps = 73\n",
      "23:44:53 [DEBUG] train episode 417: reward = 85.00, steps = 85\n",
      "23:45:08 [DEBUG] train episode 418: reward = 102.00, steps = 102\n",
      "23:45:25 [DEBUG] train episode 419: reward = 106.00, steps = 106\n",
      "23:45:35 [DEBUG] train episode 420: reward = 93.00, steps = 93\n",
      "23:45:46 [DEBUG] train episode 421: reward = 88.00, steps = 88\n",
      "23:45:56 [DEBUG] train episode 422: reward = 92.00, steps = 92\n",
      "23:46:08 [DEBUG] train episode 423: reward = 89.00, steps = 89\n",
      "23:46:17 [DEBUG] train episode 424: reward = 82.00, steps = 82\n",
      "23:46:28 [DEBUG] train episode 425: reward = 89.00, steps = 89\n",
      "23:46:40 [DEBUG] train episode 426: reward = 103.00, steps = 103\n",
      "23:46:49 [DEBUG] train episode 427: reward = 74.00, steps = 74\n",
      "23:47:01 [DEBUG] train episode 428: reward = 85.00, steps = 85\n",
      "23:47:10 [DEBUG] train episode 429: reward = 82.00, steps = 82\n",
      "23:47:22 [DEBUG] train episode 430: reward = 105.00, steps = 105\n",
      "23:47:32 [DEBUG] train episode 431: reward = 90.00, steps = 90\n",
      "23:47:50 [DEBUG] train episode 432: reward = 156.00, steps = 156\n",
      "23:48:00 [DEBUG] train episode 433: reward = 83.00, steps = 83\n",
      "23:48:12 [DEBUG] train episode 434: reward = 102.00, steps = 102\n",
      "23:48:28 [DEBUG] train episode 435: reward = 124.00, steps = 124\n",
      "23:48:46 [DEBUG] train episode 436: reward = 161.00, steps = 161\n",
      "23:48:57 [DEBUG] train episode 437: reward = 98.00, steps = 98\n",
      "23:49:12 [DEBUG] train episode 438: reward = 120.00, steps = 120\n",
      "23:49:22 [DEBUG] train episode 439: reward = 82.00, steps = 82\n",
      "23:49:33 [DEBUG] train episode 440: reward = 85.00, steps = 85\n",
      "23:49:40 [DEBUG] train episode 441: reward = 65.00, steps = 65\n",
      "23:49:49 [DEBUG] train episode 442: reward = 70.00, steps = 70\n",
      "23:49:58 [DEBUG] train episode 443: reward = 70.00, steps = 70\n",
      "23:50:07 [DEBUG] train episode 444: reward = 74.00, steps = 74\n",
      "23:50:16 [DEBUG] train episode 445: reward = 88.00, steps = 88\n",
      "23:50:25 [DEBUG] train episode 446: reward = 59.00, steps = 59\n",
      "23:50:35 [DEBUG] train episode 447: reward = 88.00, steps = 88\n",
      "23:50:44 [DEBUG] train episode 448: reward = 82.00, steps = 82\n",
      "23:51:01 [DEBUG] train episode 449: reward = 135.00, steps = 135\n",
      "23:51:12 [DEBUG] train episode 450: reward = 98.00, steps = 98\n",
      "23:51:24 [DEBUG] train episode 451: reward = 92.00, steps = 92\n",
      "23:51:32 [DEBUG] train episode 452: reward = 72.00, steps = 72\n",
      "23:51:43 [DEBUG] train episode 453: reward = 92.00, steps = 92\n",
      "23:51:54 [DEBUG] train episode 454: reward = 99.00, steps = 99\n",
      "23:52:03 [DEBUG] train episode 455: reward = 79.00, steps = 79\n",
      "23:52:14 [DEBUG] train episode 456: reward = 103.00, steps = 103\n",
      "23:52:37 [DEBUG] train episode 457: reward = 200.00, steps = 200\n",
      "23:52:53 [DEBUG] train episode 458: reward = 135.00, steps = 135\n",
      "23:53:04 [DEBUG] train episode 459: reward = 104.00, steps = 104\n",
      "23:53:13 [DEBUG] train episode 460: reward = 81.00, steps = 81\n",
      "23:53:23 [DEBUG] train episode 461: reward = 104.00, steps = 104\n",
      "23:53:32 [DEBUG] train episode 462: reward = 89.00, steps = 89\n",
      "23:53:42 [DEBUG] train episode 463: reward = 100.00, steps = 100\n",
      "23:53:51 [DEBUG] train episode 464: reward = 81.00, steps = 81\n",
      "23:54:03 [DEBUG] train episode 465: reward = 118.00, steps = 118\n",
      "23:54:16 [DEBUG] train episode 466: reward = 133.00, steps = 133\n",
      "23:54:23 [DEBUG] train episode 467: reward = 66.00, steps = 66\n",
      "23:54:33 [DEBUG] train episode 468: reward = 105.00, steps = 105\n",
      "23:54:44 [DEBUG] train episode 469: reward = 102.00, steps = 102\n",
      "23:54:54 [DEBUG] train episode 470: reward = 102.00, steps = 102\n",
      "23:55:04 [DEBUG] train episode 471: reward = 97.00, steps = 97\n",
      "23:55:19 [DEBUG] train episode 472: reward = 145.00, steps = 145\n",
      "23:55:35 [DEBUG] train episode 473: reward = 160.00, steps = 160\n",
      "23:55:43 [DEBUG] train episode 474: reward = 81.00, steps = 81\n",
      "23:55:53 [DEBUG] train episode 475: reward = 102.00, steps = 102\n",
      "23:56:02 [DEBUG] train episode 476: reward = 79.00, steps = 79\n",
      "23:56:09 [DEBUG] train episode 477: reward = 70.00, steps = 70\n",
      "23:56:21 [DEBUG] train episode 478: reward = 89.00, steps = 89\n",
      "23:56:33 [DEBUG] train episode 479: reward = 112.00, steps = 112\n",
      "23:56:40 [DEBUG] train episode 480: reward = 74.00, steps = 74\n",
      "23:56:53 [DEBUG] train episode 481: reward = 123.00, steps = 123\n",
      "23:57:02 [DEBUG] train episode 482: reward = 88.00, steps = 88\n",
      "23:57:10 [DEBUG] train episode 483: reward = 75.00, steps = 75\n",
      "23:57:25 [DEBUG] train episode 484: reward = 153.00, steps = 153\n",
      "23:57:33 [DEBUG] train episode 485: reward = 70.00, steps = 70\n",
      "23:57:45 [DEBUG] train episode 486: reward = 126.00, steps = 126\n",
      "23:57:54 [DEBUG] train episode 487: reward = 85.00, steps = 85\n",
      "23:58:04 [DEBUG] train episode 488: reward = 96.00, steps = 96\n",
      "23:58:17 [DEBUG] train episode 489: reward = 133.00, steps = 133\n",
      "23:58:31 [DEBUG] train episode 490: reward = 140.00, steps = 140\n",
      "23:58:39 [DEBUG] train episode 491: reward = 76.00, steps = 76\n",
      "23:58:48 [DEBUG] train episode 492: reward = 87.00, steps = 87\n",
      "23:59:01 [DEBUG] train episode 493: reward = 124.00, steps = 124\n",
      "23:59:11 [DEBUG] train episode 494: reward = 100.00, steps = 100\n",
      "23:59:24 [DEBUG] train episode 495: reward = 124.00, steps = 124\n",
      "23:59:33 [DEBUG] train episode 496: reward = 89.00, steps = 89\n",
      "23:59:42 [DEBUG] train episode 497: reward = 81.00, steps = 81\n",
      "23:59:51 [DEBUG] train episode 498: reward = 88.00, steps = 88\n",
      "00:00:01 [DEBUG] train episode 499: reward = 98.00, steps = 98\n",
      "00:00:11 [DEBUG] train episode 500: reward = 100.00, steps = 100\n",
      "00:00:21 [DEBUG] train episode 501: reward = 99.00, steps = 99\n",
      "00:00:32 [DEBUG] train episode 502: reward = 107.00, steps = 107\n",
      "00:00:40 [DEBUG] train episode 503: reward = 76.00, steps = 76\n",
      "00:00:46 [DEBUG] train episode 504: reward = 59.00, steps = 59\n",
      "00:00:53 [DEBUG] train episode 505: reward = 66.00, steps = 66\n",
      "00:01:02 [DEBUG] train episode 506: reward = 89.00, steps = 89\n",
      "00:01:10 [DEBUG] train episode 507: reward = 70.00, steps = 70\n",
      "00:01:16 [DEBUG] train episode 508: reward = 62.00, steps = 62\n",
      "00:01:26 [DEBUG] train episode 509: reward = 92.00, steps = 92\n",
      "00:01:35 [DEBUG] train episode 510: reward = 88.00, steps = 88\n",
      "00:01:43 [DEBUG] train episode 511: reward = 77.00, steps = 77\n",
      "00:01:55 [DEBUG] train episode 512: reward = 120.00, steps = 120\n",
      "00:02:11 [DEBUG] train episode 513: reward = 149.00, steps = 149\n",
      "00:02:22 [DEBUG] train episode 514: reward = 111.00, steps = 111\n",
      "00:02:36 [DEBUG] train episode 515: reward = 131.00, steps = 131\n",
      "00:02:44 [DEBUG] train episode 516: reward = 71.00, steps = 71\n",
      "00:02:53 [DEBUG] train episode 517: reward = 96.00, steps = 96\n",
      "00:03:04 [DEBUG] train episode 518: reward = 104.00, steps = 104\n",
      "00:03:15 [DEBUG] train episode 519: reward = 111.00, steps = 111\n",
      "00:03:28 [DEBUG] train episode 520: reward = 131.00, steps = 131\n",
      "00:03:35 [DEBUG] train episode 521: reward = 64.00, steps = 64\n",
      "00:03:45 [DEBUG] train episode 522: reward = 92.00, steps = 92\n",
      "00:03:53 [DEBUG] train episode 523: reward = 79.00, steps = 79\n",
      "00:04:07 [DEBUG] train episode 524: reward = 144.00, steps = 144\n",
      "00:04:16 [DEBUG] train episode 525: reward = 89.00, steps = 89\n",
      "00:04:24 [DEBUG] train episode 526: reward = 83.00, steps = 83\n",
      "00:04:37 [DEBUG] train episode 527: reward = 122.00, steps = 122\n",
      "00:04:47 [DEBUG] train episode 528: reward = 103.00, steps = 103\n",
      "00:04:59 [DEBUG] train episode 529: reward = 115.00, steps = 115\n",
      "00:05:19 [DEBUG] train episode 530: reward = 197.00, steps = 197\n",
      "00:05:32 [DEBUG] train episode 531: reward = 124.00, steps = 124\n",
      "00:05:50 [DEBUG] train episode 532: reward = 185.00, steps = 185\n",
      "00:06:08 [DEBUG] train episode 533: reward = 185.00, steps = 185\n",
      "00:06:17 [DEBUG] train episode 534: reward = 82.00, steps = 82\n",
      "00:06:28 [DEBUG] train episode 535: reward = 112.00, steps = 112\n",
      "00:06:45 [DEBUG] train episode 536: reward = 175.00, steps = 175\n",
      "00:07:01 [DEBUG] train episode 537: reward = 156.00, steps = 156\n",
      "00:07:11 [DEBUG] train episode 538: reward = 97.00, steps = 97\n",
      "00:07:28 [DEBUG] train episode 539: reward = 168.00, steps = 168\n",
      "00:07:41 [DEBUG] train episode 540: reward = 126.00, steps = 126\n",
      "00:07:50 [DEBUG] train episode 541: reward = 95.00, steps = 95\n",
      "00:08:02 [DEBUG] train episode 542: reward = 113.00, steps = 113\n",
      "00:08:11 [DEBUG] train episode 543: reward = 92.00, steps = 92\n",
      "00:08:19 [DEBUG] train episode 544: reward = 69.00, steps = 69\n",
      "00:08:37 [DEBUG] train episode 545: reward = 183.00, steps = 183\n",
      "00:08:53 [DEBUG] train episode 546: reward = 158.00, steps = 158\n",
      "00:09:06 [DEBUG] train episode 547: reward = 129.00, steps = 129\n",
      "00:09:19 [DEBUG] train episode 548: reward = 131.00, steps = 131\n",
      "00:09:26 [DEBUG] train episode 549: reward = 73.00, steps = 73\n",
      "00:09:40 [DEBUG] train episode 550: reward = 135.00, steps = 135\n",
      "00:09:52 [DEBUG] train episode 551: reward = 114.00, steps = 114\n",
      "00:10:01 [DEBUG] train episode 552: reward = 93.00, steps = 93\n",
      "00:10:14 [DEBUG] train episode 553: reward = 131.00, steps = 131\n",
      "00:10:24 [DEBUG] train episode 554: reward = 94.00, steps = 94\n",
      "00:10:33 [DEBUG] train episode 555: reward = 87.00, steps = 87\n",
      "00:10:44 [DEBUG] train episode 556: reward = 113.00, steps = 113\n",
      "00:10:54 [DEBUG] train episode 557: reward = 98.00, steps = 98\n",
      "00:11:08 [DEBUG] train episode 558: reward = 135.00, steps = 135\n",
      "00:11:15 [DEBUG] train episode 559: reward = 73.00, steps = 73\n",
      "00:11:35 [DEBUG] train episode 560: reward = 200.00, steps = 200\n",
      "00:11:47 [DEBUG] train episode 561: reward = 125.00, steps = 125\n",
      "00:12:03 [DEBUG] train episode 562: reward = 163.00, steps = 163\n",
      "00:12:20 [DEBUG] train episode 563: reward = 166.00, steps = 166\n",
      "00:12:29 [DEBUG] train episode 564: reward = 88.00, steps = 88\n",
      "00:12:38 [DEBUG] train episode 565: reward = 86.00, steps = 86\n",
      "00:12:47 [DEBUG] train episode 566: reward = 92.00, steps = 92\n",
      "00:13:06 [DEBUG] train episode 567: reward = 192.00, steps = 192\n",
      "00:13:18 [DEBUG] train episode 568: reward = 114.00, steps = 114\n",
      "00:13:32 [DEBUG] train episode 569: reward = 145.00, steps = 145\n",
      "00:13:44 [DEBUG] train episode 570: reward = 111.00, steps = 111\n",
      "00:13:53 [DEBUG] train episode 571: reward = 92.00, steps = 92\n",
      "00:14:05 [DEBUG] train episode 572: reward = 105.00, steps = 105\n",
      "00:14:14 [DEBUG] train episode 573: reward = 94.00, steps = 94\n",
      "00:14:26 [DEBUG] train episode 574: reward = 114.00, steps = 114\n",
      "00:14:38 [DEBUG] train episode 575: reward = 101.00, steps = 101\n",
      "00:14:47 [DEBUG] train episode 576: reward = 87.00, steps = 87\n",
      "00:14:58 [DEBUG] train episode 577: reward = 115.00, steps = 115\n",
      "00:15:06 [DEBUG] train episode 578: reward = 76.00, steps = 76\n",
      "00:15:19 [DEBUG] train episode 579: reward = 123.00, steps = 123\n",
      "00:15:36 [DEBUG] train episode 580: reward = 172.00, steps = 172\n",
      "00:15:44 [DEBUG] train episode 581: reward = 80.00, steps = 80\n",
      "00:16:02 [DEBUG] train episode 582: reward = 176.00, steps = 176\n",
      "00:16:18 [DEBUG] train episode 583: reward = 161.00, steps = 161\n",
      "00:16:35 [DEBUG] train episode 584: reward = 164.00, steps = 164\n",
      "00:16:48 [DEBUG] train episode 585: reward = 130.00, steps = 130\n",
      "00:17:00 [DEBUG] train episode 586: reward = 118.00, steps = 118\n",
      "00:17:14 [DEBUG] train episode 587: reward = 147.00, steps = 147\n",
      "00:17:28 [DEBUG] train episode 588: reward = 134.00, steps = 134\n",
      "00:17:38 [DEBUG] train episode 589: reward = 101.00, steps = 101\n",
      "00:17:48 [DEBUG] train episode 590: reward = 95.00, steps = 95\n",
      "00:17:58 [DEBUG] train episode 591: reward = 103.00, steps = 103\n",
      "00:18:08 [DEBUG] train episode 592: reward = 103.00, steps = 103\n",
      "00:18:18 [DEBUG] train episode 593: reward = 98.00, steps = 98\n",
      "00:18:28 [DEBUG] train episode 594: reward = 95.00, steps = 95\n",
      "00:18:35 [DEBUG] train episode 595: reward = 69.00, steps = 69\n",
      "00:18:44 [DEBUG] train episode 596: reward = 85.00, steps = 85\n",
      "00:18:51 [DEBUG] train episode 597: reward = 73.00, steps = 73\n",
      "00:18:58 [DEBUG] train episode 598: reward = 71.00, steps = 71\n",
      "00:19:09 [DEBUG] train episode 599: reward = 101.00, steps = 101\n",
      "00:19:20 [DEBUG] train episode 600: reward = 112.00, steps = 112\n",
      "00:19:29 [DEBUG] train episode 601: reward = 86.00, steps = 86\n",
      "00:19:41 [DEBUG] train episode 602: reward = 116.00, steps = 116\n",
      "00:19:48 [DEBUG] train episode 603: reward = 75.00, steps = 75\n",
      "00:20:09 [DEBUG] train episode 604: reward = 200.00, steps = 200\n",
      "00:20:25 [DEBUG] train episode 605: reward = 165.00, steps = 165\n",
      "00:20:44 [DEBUG] train episode 606: reward = 196.00, steps = 196\n",
      "00:20:54 [DEBUG] train episode 607: reward = 103.00, steps = 103\n",
      "00:21:07 [DEBUG] train episode 608: reward = 126.00, steps = 126\n",
      "00:21:16 [DEBUG] train episode 609: reward = 89.00, steps = 89\n",
      "00:21:26 [DEBUG] train episode 610: reward = 98.00, steps = 98\n",
      "00:21:34 [DEBUG] train episode 611: reward = 78.00, steps = 78\n",
      "00:21:49 [DEBUG] train episode 612: reward = 150.00, steps = 150\n",
      "00:21:59 [DEBUG] train episode 613: reward = 98.00, steps = 98\n",
      "00:22:09 [DEBUG] train episode 614: reward = 102.00, steps = 102\n",
      "00:22:18 [DEBUG] train episode 615: reward = 76.00, steps = 76\n",
      "00:22:38 [DEBUG] train episode 616: reward = 162.00, steps = 162\n",
      "00:22:47 [DEBUG] train episode 617: reward = 80.00, steps = 80\n",
      "00:23:00 [DEBUG] train episode 618: reward = 137.00, steps = 137\n",
      "00:23:14 [DEBUG] train episode 619: reward = 140.00, steps = 140\n",
      "00:23:26 [DEBUG] train episode 620: reward = 124.00, steps = 124\n",
      "00:23:39 [DEBUG] train episode 621: reward = 130.00, steps = 130\n",
      "00:23:49 [DEBUG] train episode 622: reward = 101.00, steps = 101\n",
      "00:24:07 [DEBUG] train episode 623: reward = 177.00, steps = 177\n",
      "00:24:19 [DEBUG] train episode 624: reward = 119.00, steps = 119\n",
      "00:24:27 [DEBUG] train episode 625: reward = 77.00, steps = 77\n",
      "00:24:37 [DEBUG] train episode 626: reward = 101.00, steps = 101\n",
      "00:24:50 [DEBUG] train episode 627: reward = 126.00, steps = 126\n",
      "00:25:01 [DEBUG] train episode 628: reward = 105.00, steps = 105\n",
      "00:25:14 [DEBUG] train episode 629: reward = 123.00, steps = 123\n",
      "00:25:21 [DEBUG] train episode 630: reward = 72.00, steps = 72\n",
      "00:25:42 [DEBUG] train episode 631: reward = 200.00, steps = 200\n",
      "00:25:54 [DEBUG] train episode 632: reward = 118.00, steps = 118\n",
      "00:26:05 [DEBUG] train episode 633: reward = 112.00, steps = 112\n",
      "00:26:14 [DEBUG] train episode 634: reward = 92.00, steps = 92\n",
      "00:26:32 [DEBUG] train episode 635: reward = 176.00, steps = 176\n",
      "00:26:45 [DEBUG] train episode 636: reward = 132.00, steps = 132\n",
      "00:26:55 [DEBUG] train episode 637: reward = 100.00, steps = 100\n",
      "00:27:08 [DEBUG] train episode 638: reward = 130.00, steps = 130\n",
      "00:27:17 [DEBUG] train episode 639: reward = 81.00, steps = 81\n",
      "00:27:29 [DEBUG] train episode 640: reward = 124.00, steps = 124\n",
      "00:27:39 [DEBUG] train episode 641: reward = 92.00, steps = 92\n",
      "00:27:52 [DEBUG] train episode 642: reward = 138.00, steps = 138\n",
      "00:28:05 [DEBUG] train episode 643: reward = 116.00, steps = 116\n",
      "00:28:14 [DEBUG] train episode 644: reward = 96.00, steps = 96\n",
      "00:28:23 [DEBUG] train episode 645: reward = 90.00, steps = 90\n",
      "00:28:38 [DEBUG] train episode 646: reward = 141.00, steps = 141\n",
      "00:28:46 [DEBUG] train episode 647: reward = 76.00, steps = 76\n",
      "00:28:58 [DEBUG] train episode 648: reward = 125.00, steps = 125\n",
      "00:29:08 [DEBUG] train episode 649: reward = 83.00, steps = 83\n",
      "00:29:19 [DEBUG] train episode 650: reward = 102.00, steps = 102\n",
      "00:29:33 [DEBUG] train episode 651: reward = 145.00, steps = 145\n",
      "00:29:40 [DEBUG] train episode 652: reward = 68.00, steps = 68\n",
      "00:29:52 [DEBUG] train episode 653: reward = 111.00, steps = 111\n",
      "00:30:04 [DEBUG] train episode 654: reward = 120.00, steps = 120\n",
      "00:30:15 [DEBUG] train episode 655: reward = 106.00, steps = 106\n",
      "00:30:26 [DEBUG] train episode 656: reward = 106.00, steps = 106\n",
      "00:30:36 [DEBUG] train episode 657: reward = 99.00, steps = 99\n",
      "00:30:56 [DEBUG] train episode 658: reward = 200.00, steps = 200\n",
      "00:31:05 [DEBUG] train episode 659: reward = 86.00, steps = 86\n",
      "00:31:17 [DEBUG] train episode 660: reward = 120.00, steps = 120\n",
      "00:31:27 [DEBUG] train episode 661: reward = 103.00, steps = 103\n",
      "00:31:40 [DEBUG] train episode 662: reward = 128.00, steps = 128\n",
      "00:31:52 [DEBUG] train episode 663: reward = 115.00, steps = 115\n",
      "00:32:01 [DEBUG] train episode 664: reward = 88.00, steps = 88\n",
      "00:32:11 [DEBUG] train episode 665: reward = 87.00, steps = 87\n",
      "00:32:23 [DEBUG] train episode 666: reward = 128.00, steps = 128\n",
      "00:32:33 [DEBUG] train episode 667: reward = 93.00, steps = 93\n",
      "00:32:44 [DEBUG] train episode 668: reward = 104.00, steps = 104\n",
      "00:33:02 [DEBUG] train episode 669: reward = 177.00, steps = 177\n",
      "00:33:11 [DEBUG] train episode 670: reward = 91.00, steps = 91\n",
      "00:33:20 [DEBUG] train episode 671: reward = 87.00, steps = 87\n",
      "00:33:41 [DEBUG] train episode 672: reward = 200.00, steps = 200\n",
      "00:33:54 [DEBUG] train episode 673: reward = 125.00, steps = 125\n",
      "00:34:05 [DEBUG] train episode 674: reward = 113.00, steps = 113\n",
      "00:34:13 [DEBUG] train episode 675: reward = 85.00, steps = 85\n",
      "00:34:23 [DEBUG] train episode 676: reward = 89.00, steps = 89\n",
      "00:34:33 [DEBUG] train episode 677: reward = 102.00, steps = 102\n",
      "00:34:49 [DEBUG] train episode 678: reward = 154.00, steps = 154\n",
      "00:35:05 [DEBUG] train episode 679: reward = 160.00, steps = 160\n",
      "00:35:22 [DEBUG] train episode 680: reward = 177.00, steps = 177\n",
      "00:35:35 [DEBUG] train episode 681: reward = 125.00, steps = 125\n",
      "00:35:47 [DEBUG] train episode 682: reward = 111.00, steps = 111\n",
      "00:35:57 [DEBUG] train episode 683: reward = 98.00, steps = 98\n",
      "00:36:10 [DEBUG] train episode 684: reward = 127.00, steps = 127\n",
      "00:36:20 [DEBUG] train episode 685: reward = 105.00, steps = 105\n",
      "00:36:32 [DEBUG] train episode 686: reward = 116.00, steps = 116\n",
      "00:36:50 [DEBUG] train episode 687: reward = 181.00, steps = 181\n",
      "00:37:06 [DEBUG] train episode 688: reward = 154.00, steps = 154\n",
      "00:37:20 [DEBUG] train episode 689: reward = 142.00, steps = 142\n",
      "00:37:31 [DEBUG] train episode 690: reward = 104.00, steps = 104\n",
      "00:37:46 [DEBUG] train episode 691: reward = 151.00, steps = 151\n",
      "00:37:58 [DEBUG] train episode 692: reward = 121.00, steps = 121\n",
      "00:38:06 [DEBUG] train episode 693: reward = 79.00, steps = 79\n",
      "00:38:20 [DEBUG] train episode 694: reward = 145.00, steps = 145\n",
      "00:38:34 [DEBUG] train episode 695: reward = 140.00, steps = 140\n",
      "00:38:46 [DEBUG] train episode 696: reward = 120.00, steps = 120\n",
      "00:38:57 [DEBUG] train episode 697: reward = 106.00, steps = 106\n",
      "00:39:11 [DEBUG] train episode 698: reward = 144.00, steps = 144\n",
      "00:39:26 [DEBUG] train episode 699: reward = 155.00, steps = 155\n",
      "00:39:37 [DEBUG] train episode 700: reward = 101.00, steps = 101\n",
      "00:39:48 [DEBUG] train episode 701: reward = 107.00, steps = 107\n",
      "00:39:58 [DEBUG] train episode 702: reward = 101.00, steps = 101\n",
      "00:40:13 [DEBUG] train episode 703: reward = 146.00, steps = 146\n",
      "00:40:28 [DEBUG] train episode 704: reward = 156.00, steps = 156\n",
      "00:40:40 [DEBUG] train episode 705: reward = 117.00, steps = 117\n",
      "00:40:50 [DEBUG] train episode 706: reward = 102.00, steps = 102\n",
      "00:41:04 [DEBUG] train episode 707: reward = 140.00, steps = 140\n",
      "00:41:15 [DEBUG] train episode 708: reward = 106.00, steps = 106\n",
      "00:41:26 [DEBUG] train episode 709: reward = 106.00, steps = 106\n",
      "00:41:37 [DEBUG] train episode 710: reward = 106.00, steps = 106\n",
      "00:41:49 [DEBUG] train episode 711: reward = 119.00, steps = 119\n",
      "00:42:00 [DEBUG] train episode 712: reward = 113.00, steps = 113\n",
      "00:42:20 [DEBUG] train episode 713: reward = 200.00, steps = 200\n",
      "00:42:30 [DEBUG] train episode 714: reward = 104.00, steps = 104\n",
      "00:42:43 [DEBUG] train episode 715: reward = 128.00, steps = 128\n",
      "00:42:58 [DEBUG] train episode 716: reward = 144.00, steps = 144\n",
      "00:43:14 [DEBUG] train episode 717: reward = 161.00, steps = 161\n",
      "00:43:34 [DEBUG] train episode 718: reward = 200.00, steps = 200\n",
      "00:43:52 [DEBUG] train episode 719: reward = 185.00, steps = 185\n",
      "00:44:07 [DEBUG] train episode 720: reward = 157.00, steps = 157\n",
      "00:44:23 [DEBUG] train episode 721: reward = 149.00, steps = 149\n",
      "00:44:40 [DEBUG] train episode 722: reward = 173.00, steps = 173\n",
      "00:45:01 [DEBUG] train episode 723: reward = 200.00, steps = 200\n",
      "00:45:15 [DEBUG] train episode 724: reward = 135.00, steps = 135\n",
      "00:45:34 [DEBUG] train episode 725: reward = 200.00, steps = 200\n",
      "00:45:53 [DEBUG] train episode 726: reward = 195.00, steps = 195\n",
      "00:46:13 [DEBUG] train episode 727: reward = 200.00, steps = 200\n",
      "00:46:33 [DEBUG] train episode 728: reward = 200.00, steps = 200\n",
      "00:46:53 [DEBUG] train episode 729: reward = 200.00, steps = 200\n",
      "00:47:13 [DEBUG] train episode 730: reward = 200.00, steps = 200\n",
      "00:47:33 [DEBUG] train episode 731: reward = 200.00, steps = 200\n",
      "00:47:49 [DEBUG] train episode 732: reward = 157.00, steps = 157\n",
      "00:48:09 [DEBUG] train episode 733: reward = 200.00, steps = 200\n",
      "00:48:29 [DEBUG] train episode 734: reward = 200.00, steps = 200\n",
      "00:48:29 [INFO] ==== test ====\n",
      "00:48:43 [DEBUG] test episode 0: reward = 155.00, steps = 155\n",
      "00:49:01 [DEBUG] test episode 1: reward = 199.00, steps = 199\n",
      "00:49:16 [DEBUG] test episode 2: reward = 170.00, steps = 170\n",
      "00:49:34 [DEBUG] test episode 3: reward = 200.00, steps = 200\n",
      "00:49:51 [DEBUG] test episode 4: reward = 185.00, steps = 185\n",
      "00:50:06 [DEBUG] test episode 5: reward = 166.00, steps = 166\n",
      "00:50:20 [DEBUG] test episode 6: reward = 157.00, steps = 157\n",
      "00:50:34 [DEBUG] test episode 7: reward = 158.00, steps = 158\n",
      "00:50:47 [DEBUG] test episode 8: reward = 139.00, steps = 139\n",
      "00:51:00 [DEBUG] test episode 9: reward = 149.00, steps = 149\n",
      "00:51:14 [DEBUG] test episode 10: reward = 157.00, steps = 157\n",
      "00:51:27 [DEBUG] test episode 11: reward = 142.00, steps = 142\n",
      "00:51:44 [DEBUG] test episode 12: reward = 193.00, steps = 193\n",
      "00:51:59 [DEBUG] test episode 13: reward = 164.00, steps = 164\n",
      "00:52:17 [DEBUG] test episode 14: reward = 200.00, steps = 200\n",
      "00:52:32 [DEBUG] test episode 15: reward = 163.00, steps = 163\n",
      "00:52:50 [DEBUG] test episode 16: reward = 200.00, steps = 200\n",
      "00:53:07 [DEBUG] test episode 17: reward = 193.00, steps = 193\n",
      "00:53:26 [DEBUG] test episode 18: reward = 200.00, steps = 200\n",
      "00:53:42 [DEBUG] test episode 19: reward = 180.00, steps = 180\n",
      "00:54:00 [DEBUG] test episode 20: reward = 200.00, steps = 200\n",
      "00:54:14 [DEBUG] test episode 21: reward = 158.00, steps = 158\n",
      "00:54:33 [DEBUG] test episode 22: reward = 200.00, steps = 200\n",
      "00:54:48 [DEBUG] test episode 23: reward = 173.00, steps = 173\n",
      "00:55:02 [DEBUG] test episode 24: reward = 157.00, steps = 157\n",
      "00:55:17 [DEBUG] test episode 25: reward = 166.00, steps = 166\n",
      "00:55:35 [DEBUG] test episode 26: reward = 200.00, steps = 200\n",
      "00:55:53 [DEBUG] test episode 27: reward = 200.00, steps = 200\n",
      "00:56:06 [DEBUG] test episode 28: reward = 146.00, steps = 146\n",
      "00:56:24 [DEBUG] test episode 29: reward = 200.00, steps = 200\n",
      "00:56:42 [DEBUG] test episode 30: reward = 200.00, steps = 200\n",
      "00:56:56 [DEBUG] test episode 31: reward = 156.00, steps = 156\n",
      "00:57:14 [DEBUG] test episode 32: reward = 200.00, steps = 200\n",
      "00:57:32 [DEBUG] test episode 33: reward = 200.00, steps = 200\n",
      "00:57:50 [DEBUG] test episode 34: reward = 200.00, steps = 200\n",
      "00:58:06 [DEBUG] test episode 35: reward = 172.00, steps = 172\n",
      "00:58:22 [DEBUG] test episode 36: reward = 176.00, steps = 176\n",
      "00:58:40 [DEBUG] test episode 37: reward = 200.00, steps = 200\n",
      "00:58:52 [DEBUG] test episode 38: reward = 135.00, steps = 135\n",
      "00:59:10 [DEBUG] test episode 39: reward = 200.00, steps = 200\n",
      "00:59:25 [DEBUG] test episode 40: reward = 163.00, steps = 163\n",
      "00:59:39 [DEBUG] test episode 41: reward = 158.00, steps = 158\n",
      "00:59:51 [DEBUG] test episode 42: reward = 137.00, steps = 137\n",
      "01:00:07 [DEBUG] test episode 43: reward = 168.00, steps = 168\n",
      "01:00:24 [DEBUG] test episode 44: reward = 192.00, steps = 192\n",
      "01:00:42 [DEBUG] test episode 45: reward = 199.00, steps = 199\n",
      "01:00:58 [DEBUG] test episode 46: reward = 185.00, steps = 185\n",
      "01:01:15 [DEBUG] test episode 47: reward = 187.00, steps = 187\n",
      "01:01:30 [DEBUG] test episode 48: reward = 163.00, steps = 163\n",
      "01:01:43 [DEBUG] test episode 49: reward = 135.00, steps = 135\n",
      "01:01:59 [DEBUG] test episode 50: reward = 179.00, steps = 179\n",
      "01:02:17 [DEBUG] test episode 51: reward = 200.00, steps = 200\n",
      "01:02:35 [DEBUG] test episode 52: reward = 200.00, steps = 200\n",
      "01:02:49 [DEBUG] test episode 53: reward = 149.00, steps = 149\n",
      "01:03:01 [DEBUG] test episode 54: reward = 134.00, steps = 134\n",
      "01:03:13 [DEBUG] test episode 55: reward = 138.00, steps = 138\n",
      "01:03:32 [DEBUG] test episode 56: reward = 200.00, steps = 200\n",
      "01:03:49 [DEBUG] test episode 57: reward = 200.00, steps = 200\n",
      "01:04:05 [DEBUG] test episode 58: reward = 174.00, steps = 174\n",
      "01:04:19 [DEBUG] test episode 59: reward = 152.00, steps = 152\n",
      "01:04:32 [DEBUG] test episode 60: reward = 154.00, steps = 154\n",
      "01:04:48 [DEBUG] test episode 61: reward = 165.00, steps = 165\n",
      "01:05:04 [DEBUG] test episode 62: reward = 181.00, steps = 181\n",
      "01:05:22 [DEBUG] test episode 63: reward = 200.00, steps = 200\n",
      "01:05:40 [DEBUG] test episode 64: reward = 200.00, steps = 200\n",
      "01:05:57 [DEBUG] test episode 65: reward = 200.00, steps = 200\n",
      "01:06:15 [DEBUG] test episode 66: reward = 200.00, steps = 200\n",
      "01:06:31 [DEBUG] test episode 67: reward = 169.00, steps = 169\n",
      "01:06:48 [DEBUG] test episode 68: reward = 192.00, steps = 192\n",
      "01:07:05 [DEBUG] test episode 69: reward = 187.00, steps = 187\n",
      "01:07:21 [DEBUG] test episode 70: reward = 184.00, steps = 184\n",
      "01:07:39 [DEBUG] test episode 71: reward = 200.00, steps = 200\n",
      "01:07:52 [DEBUG] test episode 72: reward = 143.00, steps = 143\n",
      "01:08:10 [DEBUG] test episode 73: reward = 193.00, steps = 193\n",
      "01:08:24 [DEBUG] test episode 74: reward = 161.00, steps = 161\n",
      "01:08:42 [DEBUG] test episode 75: reward = 200.00, steps = 200\n",
      "01:09:01 [DEBUG] test episode 76: reward = 200.00, steps = 200\n",
      "01:09:21 [DEBUG] test episode 77: reward = 200.00, steps = 200\n",
      "01:09:35 [DEBUG] test episode 78: reward = 148.00, steps = 148\n",
      "01:09:48 [DEBUG] test episode 79: reward = 146.00, steps = 146\n",
      "01:10:06 [DEBUG] test episode 80: reward = 200.00, steps = 200\n",
      "01:10:23 [DEBUG] test episode 81: reward = 190.00, steps = 190\n",
      "01:10:37 [DEBUG] test episode 82: reward = 153.00, steps = 153\n",
      "01:10:52 [DEBUG] test episode 83: reward = 164.00, steps = 164\n",
      "01:11:10 [DEBUG] test episode 84: reward = 200.00, steps = 200\n",
      "01:11:28 [DEBUG] test episode 85: reward = 199.00, steps = 199\n",
      "01:11:46 [DEBUG] test episode 86: reward = 200.00, steps = 200\n",
      "01:12:03 [DEBUG] test episode 87: reward = 187.00, steps = 187\n",
      "01:12:21 [DEBUG] test episode 88: reward = 200.00, steps = 200\n",
      "01:12:37 [DEBUG] test episode 89: reward = 181.00, steps = 181\n",
      "01:12:53 [DEBUG] test episode 90: reward = 174.00, steps = 174\n",
      "01:13:07 [DEBUG] test episode 91: reward = 148.00, steps = 148\n",
      "01:13:24 [DEBUG] test episode 92: reward = 200.00, steps = 200\n",
      "01:13:42 [DEBUG] test episode 93: reward = 200.00, steps = 200\n",
      "01:14:00 [DEBUG] test episode 94: reward = 200.00, steps = 200\n",
      "01:14:18 [DEBUG] test episode 95: reward = 195.00, steps = 195\n",
      "01:14:35 [DEBUG] test episode 96: reward = 200.00, steps = 200\n",
      "01:14:54 [DEBUG] test episode 97: reward = 200.00, steps = 200\n",
      "01:15:12 [DEBUG] test episode 98: reward = 200.00, steps = 200\n",
      "01:15:23 [DEBUG] test episode 99: reward = 120.00, steps = 120\n",
      "01:15:23 [INFO] average episode reward = 178.56 ± 22.48\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd7zcxLXHf2d3773uvWDcrm1MscEYMGAwEEJ1IKGk8ICEQBohgfSXAA+SENJIheQlwCOhJSEktAQSCGB6M8WmGIxt3O2Le6+37c77QxrtaDQjjbTau8Xz/XzsuyuNZo600pmjM2fOEGMMFovFYqkvMpUWwGKxWCzpY5W7xWKx1CFWuVssFksdYpW7xWKx1CFWuVssFksdkqu0AAAwaNAg1tzcXGkxLBaLpaaYPXv2BsbYYNW+qlDuzc3NmDVrVqXFsFgslpqCiJbr9lm3jMVisdQhVrlbLBZLHWKVu8VisdQhVrlbLBZLHWKVu8VisdQhkcqdiEYS0dNENI+I5hLR19ztA4hoBhEtdP/2F465kogWEdECIjq1nCdgsVgsliAmlnsngG8xxg4AMBXApUQ0AcAVAJ5kjI0H8KT7He6+cwFMBDAdwI1ElC2H8BaLxWJRExnnzhhbDWC1+3k7Ec0DMBzAmQCOd4vdCeAZAJe72//GGGsDsJSIFgE4AsDMtIW3WEx4esE6jB/SCyP696i0KKnw7qptaO3M49BR/aMLV5gn3l2Lg0b0xdA+3UqqZ07LFgDApBH9Isuu3LQLi9fvwPH7DYksu6u9E4/NXYOzDxnh296yeRcWrTOrQwevY8LeffDGii2YOnYg/vLycrR1FgDGUGBAR76Ag0b0xYcn7Z24HR2xJjERUTOAQwC8AmCoq/jBGFtNRPwqDAfwsnBYi7tNrutiABcDwKhRo+LKbbEY85nbX0Pvphze/kF9eAhP++3zAIBl151eYUmi+fyfZmHkgO54/jsnlFTPGb97EYDZOZ98/bNo7SgYlf3+g3Nx7+wWjOzfA1OaB3jbT7n+Oexqz5d0jXkdBwzrg3mrt+GosQMxc8nGQLmJe/cpi3I3HlAlol4A7gfwdcbYtrCiim2BFUEYY7cwxqYwxqYMHqycPWuxpMb2ts5Ki7DHwRcCWrlpd5e229pRMC67ZlsrAGBne963fZf0PQm8jpbNuwAASzfs9PYdIXQkPzhjYsltqTBS7kTUAEex38UYe8DdvJaIhrn7hwFY525vATBSOHwEgFXpiGuxWCpFe2cBS9bvMC5vF3lzyGUce7c9X+x0MoLmzWRU9nDpmETLEIBbAcxjjP1a2PUQgAvdzxcCeFDYfi4RNRHRGADjAbyansgWi6USXPWPt3HCr57Fpp3tFZOhtaN0i7qrybqaXJQ9Q6T8nCYmlvs0ABcAOIGI3nT/nQbgOgAnE9FCACe738EYmwvgHgDvAngUwKWMsdr7RSwWiw/uL97RaubiKofhft4fXo4uVGVwy12v3MvUblQBxtgLUPvRAeBEzTE/BvDjEuSyWCw1DiuDX+aNFVtSr7PcZF3tXRAux8Ej++KFRRsAVNZyt1gsFg9TXWRd7g65bPCCffn4fbzPVrlbLJaqwNQgr6UB1TJ5RgAULXcRUeFnyqSFrXK3WCxGlGJg7mjrRPMVD+OxuWvSE6hGyCmUe7ZKBlQtFoslNkxwzCxzY7x/88TCSolTMbIK07xaomUsFoslNrXkltGRxqBwg8LnLurzckXLWOVusVi6jFrT92l0UCqfO1nL3WKx1ANl0l+hpGF1p9EZqXzuIirlnwZWuVssllgwQ5VXabdMGu2n0UFEWebl6viscrdYLEZQzIBB006gXJi0HqW7C2Vyy4hYt4zFYqkKbnthKXa1R6cgUCnOcsxa1bdv3pZOv6bRQUUpd+uWsVgsVcGdM5dj5uJgXnIZUS3GtfrTIA2rO42+iKxbxmJJn660FPckOiXN+fqKzVV3rdOwulNR7hH7rVvGYklAlembukG8rk/NX4uP3vgS/vLKCqlMhX3uaVjuXTBuYJW7xWKpIopKj6+0tHDtdk2JyoRCpoHcQTDGsGFHW6ptZK1yt1jiYw338iAqPT4emJdcNZV+a0qj/YJUyd2vrsSUHz2BeavDVhqNB9nEYRZLfCrtGqhXxKua8fKVV9e1TkMeuYYXFzs52BetM19uMEoO65axWCwVRdRBor7ibgXZcle9NnWl/jeKc48oJcvLFXGcjiNwXSSsW8ZiSUB12ZL1g6gUueUurP8cKFPt6Qd0oZpyHdwFlaZyr1goJBHdRkTriOgdYdvfhfVUlxHRm+72ZiLaLey7uTxiW1Ss3rob4696BO+uSs8fWOtUmaegblBZ7rLC68prf87NM/HLxxb4209Y18V/mlWsgwHLN+7E+KsewaJ1O4qWe0F3dJAo5V5Jt8wdAKaLGxhj/8UYm8wYmwzgfgAPCLsX832MsUvSE9USxRPz1qEjz3DXK8srLYqlzhHVVVbjc1epNJUb5Jt/fxO3v7i0JHleXbYJv3t6kW/bpXe9jr+8HP9ZePzdtd5nBuChN1ehI8/wjzdaErll5DkBMhWbocoYew7AJtU+cqZenQPg7pTlsiTBmqkBKp3fpF75wUNz8fT8dQCKboUwCzVshuoDb7yPH/zr3VTlA4DnF27A1f98J7pgCAXGPOWczWQ8t0ycRy16QDWpdOGU6nM/FsBaxpi4vMoYInqDiJ4lomN1BxLRxUQ0i4hmrV+/vkQxLEDRUqrVmOJyYPu78rBxZzs+c8drAEIs9zq4+IwVzyuXoWSWez7K516dA6rnwW+1rwYwijF2CIBvAvgrEfVRHcgYu4UxNoUxNmXw4MElimERqUQeD0v9o7ursho/NPN9rk1Fz8DQkeeWO3mLWedjKPdKhYgmVu5ElAPwUQB/59sYY22MsY3u59kAFgPYt1QhLWbUgaFkqUG45SkrPPEr/yzfo7OWKT2+VQNjQN7ttRqy5J1rnKRkUT73clGK5X4SgPmMsRa+gYgGE1HW/TwWwHgAS0oT0WKJR0e+gLtfXYF8gdkOrwvgKw0V5BmqgrWu+x0+fvPMssmVBowVlXOGSPC5x7Dcq1W5E9HdAGYC2I+IWojoc+6ucxEcSD0OwBwiegvAfQAuYYxVd9dsqTv++PxSXPnA27h31srY7oAFa7bjPSlHiiUc7nMPc1VUq1smSkczMG+gOJchwQWVXrRMuchFFWCMnafZfpFi2/1wQiMtFWRPG1BdvH4HejbmsFffbgCAzbvaAQBbd3fEruvUG54DACy77vT0BKxztNEyKrdM14gUG90zUxAs92w2k8gtExXnXi4ilbuldqiH6IQknPirZwGoFfIeeknKgiqqY83W1lhx7rUGY8yz0nMZ8jqBNGeolgubfqCO8EIhKypFdVEPCqZaUBkPU3/6pNeBBqJllAOqtfWLiD73rOCWiXMacSJr0sQq9zqkXHGzluqiWhQllyJMibV25rtGmDIg+tyTZMC0lrulZKrkWa8qqkUBloOu1hk6o4Ff47BomU+4UTHV+mvobhNxhmqGRLeMed1WuVssZaBalUkaVEvHpbPcyyFez8Zs+pVCH80jxrkzsIQzVGNkGUsRq9wtlhqlQgahlqDlHo+wJF/D3EioQ0f3jyuWETpdzVBMH1AoCCl/ayAU0ip3S11TJcZtWaia2HFXjKDlrpAvROQw5V6utLhR90eBFePcC0y03M3bqLn0A5bqo0oe9eqiiy/KrS8sxbm3dM2sy2rpuHgnU6r3gc9TCG1LOOdv3/sWvv9gaVkfvXpD2uOWN0OylZis5W4pGW4p2WCZyvHDf7+Ll5ckn5T9+orNOPeWmWjvjNaUXa3cdbeVLswxrnyDezVp9/G6xbeVe2e34M6Z6axdoB+/KFruTLDc44x3VKoTtsq9DrFZIYtUjevCkO/cNwcvL9mEZRt3RpaVrcdCgeHKB97u8vQJXIxSo0L27tfduK200VXrzFAteJ8zCaJlKoVV7pa6plpcF6Z4b18mZaXvSzbsxN2vrsAlf56dulwmcphEy4T9HGGuDr4n7d/Te8vVDagyYHcHV+6suF5sDdxYVrlb6prqfwT9eLOMDXxrsjLs9FLTdu1j7blMSrzYYZZ/MTdNeX5RbSgkGHa0OjmKRPEqNUgaB6vcLZZqwtUZJuMmsn7p6HRnUma71i3nWe4hk5hMMMkqWTa3jG4SUwHY3trplinmmakB3W6Vez1iB1SLVMtEHxMefWd1rEyW8rl1uJZ7rostd05AuavcMiG/R1jseLmzSurj3Bl2tDnKvVAodlcqWTvyBdw/u8U4Bv6io5tx7PhBScQ1wmaFrCNqSI91GZW6JIVC0T9rwqotu3HJX173vpvEdcs6hE+2aSyX5a6p1kscVmJWSKNQypR/0Mh87gzY1e7kxSmw4jmK176tM4+Zizfi7Zat+NWM95DLEs6cPDyy7WvOmJhYbhOscq8juF1hDfdwZi3bhMNG9089wdpyIcIlzxgyMX6JNin00WhAVbbcXe2Yy5TJctcqQh7nbjCJKQSjAdWy+dzViDHqBcaUHdlPHp6HO2cux4RhznLRvDOoNNYtU4dYt0wRWV88+s4afPzmmbj71ZWpt/WBXzzjfY4bFigrQpPfUG6CK/eGXPGxbu3IY+WmXbFkiYvOco9LqHJn/r9po+uIdrj+dt42L9chvGbMX+OEnm7a6SwS01ght5hMdUhhSQXrlgkiW3orNjnW9ZL1O8rabldkApTPrcN1yzQI7qAv/GkWjv350+k0qHPLuH9DFmIK3cYJv2ZM+F/ak8KNr7fci0q8wIpXXJxk1i75kxpz1aFWTdZQvY2I1hHRO8K2a4jofSJ60/13mrDvSiJaREQLiOjUcglu0WPzuVeeUqecmxwu6zSefVCMlnl+4YaS5DBBN4kprs41sdxVZdLoR3VNi+fEhPZ3tneizc1R3+bGwfPHTqXcmyqg8E1avAPAdMX26xljk91/jwAAEU2As3D2RPeYG4moPDk6LRYTZGsyRqhhKZS64r2JNRoIhXTbVMW5lzNqiNuzwXOOGQppcM3iRuCYo65Djm3n3x95ew32u/pRAEXLnSt+lXLv3a3rhzcjlTtj7DkApskyzgTwN8ZYG2NsKYBFAI4oQT6LpSR06qbcbzddYbnLVmxHp34SUzlddp7lnqBDEgmLlmHC3z8+vwTNVzwc2FcKRpY7U5fjLppWbsEr6undraFUEWNTyrvCZUQ0x3Xb8CTLwwGII1Ut7rYARHQxEc0iolnr168vQQwLZ093uX/i5pfwX//XNRkZZWTrsdTBRSPLXfrO/cM5RQimqTRcccYZM+DnWl63THFE9Y6XlhkfZ4quBvGcCgWm/F2Kyj3PRQwwakCPkmWMS1LlfhOAcQAmA1gN4FfudlWnpbxujLFbGGNTGGNTBg8enFAMi4jnciixns/f+Rr++963Spanq3lt2Wa8stT/ktlVg8zyoFpcyz2JQ0N2g/ABVdUkJlPXxc8enQ/AP5DI0d1XxdjvUuPcTUIhgy61Un7jqNBK8W2kwNTnxH97Hs6qOo9PHzUa3/3whOSCJiCRcmeMrWWM5RljBQB/QNH10gJgpFB0BIBVpYloiU2J2v2Jeetw3+yWdGSpMPLDG6cDfPa99Zh+w3O+sDcd/JWcU6rPPYk1yuVUTWIyrS1JyKG3UpFmfMOUh95ahV8/viA1ueKgTz8gKnem/F3l9Myq365bQxYfOzR6YlOaJFLuRDRM+Ho2AB5J8xCAc4moiYjGABgP4NXSRLSYUmvpbbsCrTIw0O5X3D8H89dsx/rtbZFleeQEp1Sfu4kSCyQO46GQJfjci2GN5vJzS9Ukt0zUPfrbpxZFyMcCKa3Tccuo6+j0+dzVpeTfXiVPQzbT5am4TUIh7wYwE8B+RNRCRJ8D8HMiepuI5gD4IIBvAABjbC6AewC8C+BRAJcyxqpjutYehM3n3vW0SZZ7qXHuJgpLLtKe1+eWiasA48gfpyNbuWm3l6slDmLmyVLdMttaO/Cd+97Czjb/BCUVfss9eB2veWiu5w4Ty8k0ZAnUxdGQJtEy5zHGhjHGGhhjIxhjtzLGLmCMHcQYm8QYO4Mxtloo/2PG2DjG2H6Msf+UV/zaYnd7Hj95ZJ438FLrtGzehRueeK+qk3MFfdnmssY5LTl9QKnKvRTLPUZKG0W7QRfLn19ejjdXbjGWo1iXuvyfZi6LL1dInXGv9M3PLMY9s1pwpyCHdkDV53NngfblwV1A/ds3ZDNlWwdWh80t04X84fkluOW5JejbvQGXfnCfSotTMl/882zMXbUNH560N/YZ0qvS4ijRLf1m8nbj5eoxeCblDrtrlLv8nSt3hc89rltGqPy7/3S8ruMG91Qe05mPp9wTwYp/5LOLa1yoQjZ1dXRKlrtJW6rOrjEXJ9NQOlTHPNk9BD6DUPcwlEraBnRbZx53vbJcOzhYVGjVa7nrSNuIki33Ge+uxbIN0UvlceTfzsyNIndcrusiga9blkPVvm5ugNZyT/G+KFruwTpj96MxOviCgc89KE91WO5Wudchad1Dv31yIa76xzv41xx1wFMtqPQ0OjwTJSBH1Fz/xHs4/pfPuDIw/HvOqtCom8AbhoFcQcud16Wq36BCgTjWre4tpVzeukAnY9hOW2cej7y9WpjIJlShqUOXfiAM1c/ckKUuT+hnlXsdktY9tGmns3BEkgGwNGjvLGDm4o1lbWPr7g68sWJzaBkTey3soX9s7lpc9tc38LuQSBD56CQDqqpc42L9r6/YjG2tZouBKMLctZQaGWRC2FJ+b6wM//04v3h0Ab581+t4wc23Iz4nxZWe/A0EJzFFtzNr2abAM9OYzVjlbklONQ9sJuEnj8zDeX94GXNXbY11XNh1kBeg/uwdr+HsG1/yXGbqY0waVW9+f8tubN7lpIJdu61Ve3hgAlCCAVXPclcIs6utEx+98SV8WVgQJE7dgN4t0zUZMB1Ucl10+2tGdby/ZTcAeKtdEak6SP93eRKTyan+7bWV+PJd/utclaGQltqj65NClqfBheucPNmbd5ovPQfIscnhZXkUSFgxIytas33adU8ZySLvS5I4rDhFP1iWK7QFa7dH1gukEwqpO4UkSk48tSR32+qtu71xkWIHX6ypOEkqxHJnDKY+oHfe9xskDblMSVFMSbDRMhWg/iYblfd84l4vccZg2OSU7a0doa/7cWZFhnUAJs90kqn7QcudD6gG2e66CXo1mT3yceLidQPuUb+brgNjjGnfEhhjibT7UT99KrBN5XMPuMekAVVTd5V8bo7P3YZC1i9l/nHrzCuT+DVWHLgMWrfO31uec8JSuTUVpsyS+L99+yKPVrgHErg6+CGqY/mEnZ5NZhm44zSf1Oeuu2adBYYGKYWC5xNP1E50x6lz+wRzyyQ714aM9blbapLq6rTkXB++umK0UXzgo9sstV+N8v2qiGW5u8vF9Ww0s+fiuGXyBbEzNXeJ6TpN1e/nFU1guOty3oihibo3uEBumYQ/dCbT9Za7Ve51BL/v0hu4qeyrAH8W4kohZmiMOjZsoM4rk3DiShwCCc6E7535ApqveBi/f9ofbaNTWipRtrk+956GbhnVOevuKnEsesyVj+DhOc6E9TN//2JoGzpFKc8ZAIQ499Aade2oj/K5ZfjfiAHVuJPBVOgmg6VN3Sn3Ret2oPmKhwMDGnsSe/oqez6fe8TTGDZphyMqobbOPJqveBj3vCYtsB3SjJnPXS0XALS65yMrd9m6LlruQWF4CKSpcjdZeENul/O311aElqcIV5iciAuAd33D/PGm8inxfO7+soHEYSV24nd/YSru+eJRJdVhSt0p9xnvrgUA7cSbeiZ9n3vEQ9RFhn3cByrs1VlXVVgLYvs8cueXQmraj9/0Em56dnEcEQOEhUKSYpt4zEm/fhZ3v7qiqNwVJ1N0y5j53ONFyyTLq6P1uYfM4GaI7ixl3S8PgnppJRTbot0ypZ3bUeMGYmCvJqM6SqXulLtHFQ8ulmvgk9949WK4J/VRig+gfKl1A2JM4aYvWvWiTMFys5ZvxqtLTVeiVKNT3GF05h1LctG6HbjygbeFGarBY9tDluAzkScMWZlHKXd+CXXnqNquU74myAupeHIQRU4eEw/VLdZRrdSdck/qp+0Kyq10S/FLJiLihF5atAE//Pe7JTcT93xE3WKqDOJGyyS5xmGRFmERHaK7RSyXL7CA20AnGy9n2l/+71ML8dT8td7KTGHIyvyVpZu8N+gwdNdc1TkUQxVZbLejPEGN12USCukfLE6Qx6aC1F0oZL1Yraa83bIVz763DpedMF5I1xq8A2cv34TZyzfj4uPGpdc4C3zwcf4fXwGAxMuLJf0tk/hFTZV7EplMlFGYBSnmjBHFzDPmG1/gSlF1KnyfqfyPzV2Lx+ZGK2hAHQr5hT/NijxOpyjDOlOTn5bgv55/mrlcWRcptoUNqMbxuVfDbPG6U+57Gh/53QsAgMtOGB/qc/3YTc7C0fGUe2n+xeL++INgCcTQyKP3ZZs2ETbBKS3k2HSfchCsSrFUvlDwYvpzGSrGuSuE437xUn4H3aFx0w94b9dayz24zbdYR0QXRVJegd88uVCqSxJE2BacoVr8rMrnXs3UrXKvhp6zq9G9WpZK1MMU1Z5qUopRu56LzdydAUg+d2m3riZjt0xgsC6dqx0YGxA2FIQfVjzfzjzz+dLD4tzLmf8lad26w7oiVw3g/ylfX7EZIwd0x7urtkmyFLV7KZOYKkGkciei2wB8GMA6xtiB7rZfAPgIgHYAiwF8hjG2hYiaAcwDwEMJXmaMXVIGuUPk7crWqouwlK+lED2NPPz49s6C8UCeiMlPqWo7Scy5+hDu5tLv64iTPjEEXRIwcR+TfokCY8LSehQa556P6XOPQ9IY/3gDqqW1papNvBb3zW5RLgovWu7LN+7E4vXmOforjckTdweA6dK2GQAOZIxNAvAegCuFfYsZY5Pdf12q2EWq2XDnos1evhnrQjIFxq+X+f52FVEPXFgecxPCqlft8g2oGrahVCiqV3WpmLx+ZmLkNwyFz12UCXDeiFSWu+qs57RsDRwfW0TNsXEXnwlzHwHhA6qrt7Zid8gylY/NXRN5P3oDqiaLdQh1vdWytWLpr5NgsobqcwA2SdseZ4zxs3wZwIgyyJYI/oNVo26XraaP3fQSTrnhuVTqZqI/MLWTNzPzdM9S1k3cogtFi2zdwMxUuWVCp8BrhA3TB34r2l8+LFVwHHQLbzhtFf3NYsedLwiWu+hzV4g0f41ZNsgwdJcorhuFl9e99ERNoFqxaZd23xf/PDuyA9u400nBbPIWk9RFVA36J41QyM8CEBfCHkNEbxDRs0R0rO4gIrqYiGYR0az169enIAavN7WquoQtu+Kls9Uhru/Y1TeW7k0h5yp3bt2u2dqKTe6DFav+kix3s6thmn5Ari9pxxVoQxGRX5StuMUXLVNg6Oh0Nvh97vpzWbutFW+u3OIlEoslo87Sjvk6wMcpfFFIFNwf1W6pEBD5sMgTtGqJkpQ7EV0FoBPAXe6m1QBGMcYOAfBNAH8loj6qYxljtzDGpjDGpgwePLgUMSxwHoCwrICceAOAZmV1z56n3F3XwdSfPonDf/yEcevezMyYbafnc+f1BT/zTWmthxtmuRejoPyFOgsM7XnHReH43IsWPgDlqkv/eWcNzvr9izj/Dy/HljGXUauLKOv2wOF+FcA7A/F3ahDq7qoBVZM1TVs7nHs3W0Iy9lKOLYXEyp2ILoQz0PpJ5t5VjLE2xthG9/NsOIOt+6YhaFyq2edeDpxVYqIt9ySKLzJaRqfc3UFU0bpN+8FVWalhk5h0rZtGyzBB0e5q7xQWCTeQNewNJDCgygKfVZZ7u2u55zLkuTl4kUnXPK5t762W+LmXOgoFHN7cP7A9qoOTB9O5gSGeS06IppLfBMr2LBvo3N3tzu/bvcEsbYOHIHOllHuiUEgimg7gcgAfYIztErYPBrCJMZYnorEAxgNYkoqkllCc2Yvu55CHIc9Y6vGvUW6ZsBS8JbetajosFFJzbXTrjjr7gvUxABO+9xgmjegbKaPJwF2YnOLvqvO5i26ZdCJKgixZvxPjBvcKbI/qsLOShawaUM0JCjCYC6Zy7HI7724NGexoS1aHfP5dhUko5N0AjgcwiIhaAHwfTnRME4AZ7qAXD3k8DsC1RNQJIA/gEsZYaUk3ElLV8ahlePiYz+ce5pZJvWltvDK3xjryhUTx4EmfiSQvB+Hrrqo/A8UolFKRfzNdrH7Qci+GQnrnXcZbX/WTRPncZfdH0S1T3CZa90HLvYw+9whaXcu9KRfPchclrlrLnTF2nmLzrZqy9wO4v1ShSoFHWFSjW6acC+Q6Vl3xs8jxv3ja+xx38AvQdxbF9nSWu/PAduQZdrYnDyEzVbwcf+Iws/MN6xBULpK0Ji959Uqd7tf+9ibOnDw80L7YameBeWGmDdlMlwyoqzrcSFebZuKXz3LPUmA/p1zn8+375kSW2S1Y7kmpkG6v7cRhc1q2YP/v/gfPLFjnbauxYJnUCEtHumxjMXQsns/bMBRSs1203Le1JlHu0WGt8X3umo4qJKRSFTef9pif6TiJL/5djHPPlN8tA6gHIaPuKVm5FXPgiG6Zrh9QNaGo3ONZ7mI8fM0NqFYDjDmj2dV0M1QKcbAtzNItFBhWbtqFI378BFaGxAsXa9VTzDOuLpcVfO7bFZEbURgl3FL5ymNa+kBUDvig5Z7knjNV4MF96jo6C/4ZquWaoSySxHKXlZvKLdO7Wy6wn1PJt3A+oBpXuYtY5Z6AYgKiysqRhLT9iOL6jmHPWp4x3DtrJdZtb8P9rwenW6vQuZN0biAAuH7Ge1jiTtVuzxe8xSKSED/OPb57QmXRq86Pf46l3BN2UsV9av97vlDwIlWyGUp03nFR3QvRlrs0oKpwy3xiykicPmmYbz+nkuNn3HKPHS0jYBJyWQ5qW7krXttrZRJT6q/1xgOqxb1pjQGoWhMz8XXk/Zb76ys2G9Ub9Wag26dLkRtG2ECzyi0SZ3KLSUce+rYl7vIpd03IZFlN9+CmqGshzzT2LHfhsHhwXLIAACAASURBVIYs4Wsnjvft57y0eGMCQdOBd1yl+Nyt5Z6AqNSh1QpD+n5RMf1A1IQcvj+tjjBqcLEjX8C23UXL/aM3vhSv/sSWu94a1x0T1X4SnzsP85+7ahv++Lw6MtjY5+4LhSx4+4iEOPeu1e0JfO7O3z++sEQqR8r6PnP7a7FkLAdN1i1TGXyWO99Wpjv8Ly8vx/KNyTLD+aZYp67czdwRSaJlbn9xKd7fslvfdsTxSzfsCizubELUQsqALs69uM/0lT68Q1S7RUzhaWPnrd6GHz08L7KNMNnEz50F5inCDIV3ammRUyiquKGQHfkC/vfJhXjg9fd927kSLOeAcFJKcctY5Z4Alc/dC4UsQ3ud+QKu/uc7+NhN8SxPFaXcv88vXI933vfHVxcYU/qIZRy3jGvtGba3cN0OXHrX69r9Uefy2ycXYuG6HYatFeFuo9CHXbEriTI2zRmfRPGY+OeNOxexXmGchYQ6yqkbVfHeKzfpO34gaLn/e84q/GrGe4FyWc9yTy5fuSjJLWN97vEp+oy7pqfnrWxOIdlXKQ/gBbe+ig//7wv++iAqAX3l+QJL5JYxTayVhJWbduHlJXq/arhuD/O5K9JxaeoKj3M3k0WHvAxd1AIjMn7LXXDL5JnnEiMSBlQZDCKhkpFNtOhK8ZheTTkvX4sMj4YU3XxJoqzKQbeYk5hEMtU6iama4TeD6rmYWYZBmDQtorRfPQvCLKawqvOCwouz5Frf7g3afaWeybE/fxoAsOy6033bk7plwpYb1FnoYfncVYOWcZAt9wIDTHRkocCQEaJggGAoJHeJkCBbgTF8QJi4liZJrFDxmDA9x90Xopvnm/e8Fbu9clBSKKS13ONTfG0XtrnXceG6HYl94zrS9GWm7ldk/odb265gucdh2cadWgu9s8Dw1sot8Ss1JCwXudLlzor7AjJrzl0uJy6GXGpkk2y5q9w0qt9M9Xv6QyGLE9eI/BkryzX1I4n/WNRtYStyZRUDqovXx3fnlQPRLaMadwijUpZ7bSv3iDU20141pejTLP3JSfrwrdioft12skI6n6Mt96JCMGXlpt24Z9ZK5b7rZ7yHM3//It5286yktYAFl++W55Zo88CHuTiEl5nAvuAxcr3Fzxt3tGHLrvbQ48OQo4nC3hJEVJN9fNEyjAl1B1P+xsXkvk6i3MUB1TDlnlEMqKaVUrlURMs9F9M1lWCFyVSobeXu/vUNqPr2p9tjpmVsizHpcVi3vRXHaV63xfU1w2rOF4oaL+71WSQNivJzmNPiWO0b3LR5JgtYdOQLnsI04fmF6gVdVOcaGgqptdylyULCvh/8611MvnZG6PFhyJa7/B1Qd/bF0EafdvfI55k3+JghsY5kN6rJuSUxQn2We05fgcpyL3WJxrQQlXuDJq+9DuuWSUDRcldsRDzL1IRSXSmiOCaWe15IDAWEr9rky+ceUrdozRI5VrappR01GMmvd5tmwEzk8vvnYPK1M0Jj5MXOh08DN5EpbBKTrrkC87urdJ1vkntAnuSjXiM05A1ELOer159PqOjGMZettSMvuJ+cv2HWeTamYpPrM7Hc/co9+mTiukmSkMRy58+DdcskgmeA7JpXNxPL2BQTJfGxm17C+Kv+E1kOcK+B53PV1y0+OARg8rUzMPWnT/nrSQh/iNsM8rc/+OYqR56w9oRnQmXtAjrrXFSK0S4R1XYWOLK4PS4mPvdQt0whqMD5Z7EDEKOETNjV3on9v/sornfDEvnx4crdqGofYm2NmgoYU8e5m8wE7grlKca5m3ZwvNOxlnsCoq5Z2tc0zU5Ep2Sar3gY1zw0FwDwZoxBSiZY7mEaKC8MFALOuMQGYRWCUk6R38RxFucwsZaBkFhxlVIMGXzQtSH75/Xum/gXKC9Zn797ahGar3jYd52UfvgC36eWS5zEJLqVTCXcuMNxi93vTibiMoRZwkkUlehzb8yZDKgWt3UY3EtdoTzFAdUGQ8udh09ayz0B/KbRPW9p+9xTjUAIqeuOl5YFtl1w6ys49xb9upeiCyJMTDFaRvVMhB0bdfoZz3KPXnqOK6V9ry6+mcjXVxRPb7nr616xaRc+dtPMyPKAyueu7wTiIr+d3PbiUgB+V9MVD7ytPU43FuCkH3A+PzFvrRdVZCojXyKwe6OjhEzcMkkUFflCIcPqhk8OAOgweOi6Ygao6JYZ3q+7b98fPz0lUP4rJ+yDb53irDBqLfcE8EvmW0Vd3J/yNb3g1lcApDOwGrejeH7hBm3ECOAoI5NQyHxE4rBSxhXiuGVUhL2C5zX7wnzuLyzaoNind8uYWO5JOnjdW0dbPrwT5Mf5Lffil0feXoN/vbVKIaOZkPJCFLydtC130RMTJptqQNVkPKgrdKeo3K85Y6Jv30kThgbKf+6YMZiwt7ME48BejeUVTkNtK3dF+oFykuaSaulPYooRChlmuYccGyXyTc8sxtX/fDuxcpeVoGjxxfG5q5Qix3RAVUcSt4xO9vbOAh54vQVf+NOs0LbC0imE5fyJgs8U5e4Dft3CfMrcyo+DaK2HXT5uHPx6xnv45WMLAJh1pl1juTvXpF+PBhw4vC++efK+kccc3twfPzhjIn7y0YPKLZ6SSOVORLcR0ToiekfYNoCIZhDRQvdvf2HflUS0iIgWENGp5RIcCKb8zRcYrv7nO8L+6iIqcVgpPn3n2Gifa6EQPuBaykStp+avw19eXmHkllERUO7C51nLNuObf38zmOs7ZCAybHWl4Hb/uQuX00ciy10T8dHWWcA373kLM95dqz7OQLmreH5h8I1FBV/6kFul/NqEWe4XHDXaqG4RsZMOu7/Ecr9TJJo7+5DhyuO6xufudoDub/mpqeHXIV9gICJceHQz+nTTz+4uJyaW+x0ApkvbrgDwJGNsPIAn3e8gogkAzgUw0T3mRiJKPm83Ajnl7yrJiqnm3O5h7oQk+C33YkWyMvRb7sEL5B/glPYZKv44A6o+2UIuwFPz1+GBN94PLPqhOiJsZqne3SKFQmrONknnp4sIigoZVb2JxckjH8XONq7c/W4ZnSV83hEjYy8UDfhj40t5YR0zqCf236t3sP4usNx7uG8s/C1M16EM7dOEyz64Dwb0rIwrRiRSuTPGngOwSdp8JoA73c93AjhL2P43xlgbY2wpgEUAjkhJVr2M2j3hPzpjDDc9sxhLN+xEocDw68cX+CJHyknYdPNkFKNgxFrkOnXjE14tBm4Zxhj+98mFWpdAcp+77JYJliHpjlVZ4qo1Ojm6axzoyDTXIdGAqs4tE+FP5h2zP4d7eu68PzzvDOx6Vqlbty6OO04uIhHRLRNHfPn3KzCmHJD96onj0bupfGmyGrOZgOtKvg+/eNxY7NWnG/528VH471P3S3yt0iSpz30oY2w1ALh/h7jbhwMQ56i3uNsCENHFRDSLiGatX6+efRiFd/3ce0B+cKOu74Yd7fjZo/Px2TtewytLN+G3Ty3C5QYroqeB6iYv5cEVLXffLEb5ARHj3JXRMuH7AWDphp341Yz3tBNMkir3qEU/VIS9AcV5Owpa7ubtycjXTedzb+swHFAVLud9s82WRoxirz7dvHxAsltGZ7kndX/4LXfz31hOH1JgatkG9WzEjZ86NJFsJjTlMl4IJ39zkjuZK087AC//z4kYM6hn2eSIS9oDqkpjUFWQMXYLY2wKY2zK4MGDkzXm5W7nllq843koWke+uKLNLs1MyLQRO6KXFm3AwrXbS3plFeO0A75jASflr76hxeuik61F6eCkbpmA5a4oE+caqYqG5ZbxXzdduWgB5AdfF+kT1Qmqop/+7zn1Sk5xWbOt1fvcwwuFdL7rfO5JBy6TWrGbd0ozspk6EIFI3/EcO35QaBsfPVTtxxdpzGXQlJNcV1VgmUeRVLmvJaJhAOD+XedubwEwUig3AkAwVisl+L3Gf+/ALMOI53BXh2MZdG/IGqWXTcp7a7djgZDZUI6pPv+Pr+Dk65+LtUqSfG85lmewkwummxVDIYN85HcvKLaGty3z5Dz1AGEUYdEyHJMBVa+saqfW3WJoueub85Cl1rpljJW7QaMpwO8/XbRMUn3md8uYn8y67a2+7wzOUoUKybQdSNTb8K/PmRwpR2Mug1w2g4Ys4bsfnuC0WP26PbFyfwjAhe7nCwE8KGw/l4iaiGgMgPEAXi1NRD1yyt/gzxj+w76xwnkt7dGYjZwQVQqnXP8cTr3hOX/UgNJlUNy4ICTNrQrG1Jae/DDlC6WfY9R9/Z931iSq18QtFUwToD8m6hr765FmqBbi+exFgpZ7PPfV/5y2v3ucK0uZY32L7h/X515Gt0ycjmrlZn8G1LBrr3upSCOrJH9jWfjj0/C5Y8a47VW/djcJhbwbwEwA+xFRCxF9DsB1AE4mooUATna/gzE2F8A9AN4F8CiASxljZfNzyCl/TQfFOFe6swK7NQjKPcWc7ToYdFkAixtPveG5WHWKPmPxvGWPgJjyV+cL9uQMXM/yXhsTt4xYZOWmXb6Ft4Nlg/LqnnXxzQcIuQ8S+NznrVZ31LqQUZ5/Ja7l/tfPH2lWUEJOOKdzv8hRKfLiKjrE4/ShqMHt8vJ9uutApJc5jegiVWdXqXVR4xA5xMwYO0+z60RN+R8D+HEpQpnijad6P7ps1ZnRvTHr9fxd9QqsjpYxP16t+NyHNKQdMf2ASWpeZdtlslpkC1d1OUQlwFdw0iEfv2lnu3bQduOOdixZXxxvCAuZjEJ+8HVRRTq3TJMUvWLqykgSEtiYy3jWbVRumaTWqnhY1JmcdMAQPDFvHQb1asTqrX63TNh1SOqWMSGnSHZWA7q9tmeowrPcHcIWXAijR2PpPvfWjryXq8MEpVVZUrSM2nKX/fhi+oG4A59hvvo0CCj3EjtA+fhDfzhDe42//9BcnPn7F4vHIqgwOvMFs2gZQ/l0bhluuevGknSYKl8x8VWvppx3jxR97jrlHtz21Lc+EEsu3anwa33Tpw7D29ecgsZsJpjLXXesRjYg+u3UBFVnVw2hjlHUtHL3cqMoBhIBcxdLj8ac92MlvRf2/+6jOPq6pwLbda+hpjm9dcg3lzNI633ztqvi3PmmqGgN+f6Nq2ziolprNFAmRttxfO7BY1lAYVz619cT+dx16FYKa3InFcU5V8A8HW9OGDDt0Zj13ma4Ba+Lc1cp/T4ha+tyxMOiDJiGbAa9uzUgmyVlMIAOXYeUhuVeCy4YFbWt3AOWu2z5hR/PczR3a8gIA6rJbwZVYq+v3P1GYBtjwBm/ezGwPVa0jPTdl9s7xOcuXqOkIYvlcl3J/lFVM3Fi4UuZKMYQvH8em7vWyFwwdY/8ws2fItOUS+aWMbUmufJuymXQkM0gz5zZ3XycRx8tE6xftSrRWZP39n0XO7uJe/cBAAzp3RQuYyajeJNTlyUibYeaJGnXOVNGSLJY5d7lyD73uAOq/CbPF5jgc09Xc/17zmrjsqUoTQYxzl2sU3bLFEvc+sLSmG3wzqM82j3YOavebszrUxU1teQYU3e2Rm6ZEnUBnzDjpVEw7INNo1m426dbQxbZDKFQYD7/dpxomazCyp86dqDvu6h4f3XOwbjvkqMwrG+3UBmzGZXlri5LCF7zhizhrs8fiWP2cebQHDqqX2h7IqMH+iciWcu9AsjWtuyGiXLL5KXXUacuddkksyfjEqeNoMtEnfJXfkBKdWvErSMOcthaKW4V3fGmkXG6zJ0mHVupYXJcucYeUI1puXdvyCJLhM5CAb2E6ftxfO6qjqCnlApAFKtHYw5TmgdEvt3kMhR4k4vjliEQpu1TnMB0+JgBoe2JDJTywuQSLC1YDdSm1C7FQVDnb1zLnQ+25Bnz6gibnl4qUYohThtyLnbR5+7P/y21UWCxrF+Rv7y8ArvaO40syST6LRgt43zv36Po1z3+l89g8Xr/Qt1xMH7rYJrEYwaHlqrcM9L4j6nIpjqIr2PavTGLTIYwc/FGX+itNlpGsV2t3P3Jxfj5NAmrMEW5OjIUtNyjykdx/5eONqprqPRWYS33CiCn/JWV4zn/NxOL1uknA3FLOV8Q/dWaAVDDJ+z2F5fiV4+rfam8Ct0CBHFfDvjal/zY4mu8YLmromVM9Zui3Pw12406oSR+yj88759aXygA+w7thWYpX8fdr6yIXTfHZPFugPvck/nsS3XLiC7C215Yiu/cb5bvyLRT4ekGmnIZZDPANinTpk6ZBaJXNGV7NPotd16kd7fotwNOLkuBSJcCY7j9osMDZYmC5656az9sdP/ANhV79fErd1HuWqKmlXsxWKaopEV2tefxk0fmaw/nN09nQR1GKKKyVjfuaMPX/vYGtrcWc2D84F/v4n+fCuaiBoqd0LbWDuX+WCP7BPzmyYXFuhlTdnKqaBnTjkq3EEaSWG8Tnl7gTyDHwECgQEdRivKUI1R0vt8fPzwvVspgERPxLvvgPtp9WcEtc+2/3zWozX9cFL3d/OIZIuXgKb/e8nXe0RqM7lENsvaUlPtQV1mKnUhDRGhPNkOBAf8CYzhC4V5xlDukss7fJJMSh0rKvVIrKZVKTSv3YH6VYJndmkRgPus2X5yduGDtdry8ZGOgvEoh/nrGe3jwzVX455vq9Dn3Sxn8eBVbd6uVeykDlU5WyGInt3VXB655aG4g9j5fYAHfvs7KVonTmWdGbxhp+CkZU88+LCXGmC9QwdFZuw+9tUrZ2X7fXbw8DBML+tjxg3DwSPUgH3d/xHUFmvan3M2VyQCqqEeu8OX7Qhe6KbPPkF7eNH0AmOguNycq659/fFJoHTmlcteHaQYsd+namayn/O+vHIOLjm5G/x4N+OqJ43H8fs5gbDXkZk9CbSt3929Y/HWrZop3p+S6EJ9j1ULUqgedLxzRU7H02O72PL5171u+bdyK0GWeLCUUkoF5bxcFxvCLx+fjjpeW4QF3ZXtOQREFomtXF21SLstdpsAcRS53FH+euRzz16gSSEWzU1JQUXpYXuk+bB1b0zoBoCGnf/S4onp6/jptGVUbpm6ZgT2bvPKq34kb1fK+7YbKPZMBvvvhCRg5wFlIuldTDp8/ZgxuvbC4kPSwvv5FpmVlnCHCrOWbpTLq0EuVrPIdGmbBf2rqKPzx01Nw4PC+uOaMiSAifPPkfTF6QA8AxetVa9SmM8lFTvmrijbRWe6iso5Kg6urmyuKRsWDKme0A4AHXQtf537RDVSqZFNNMOKl8gXmRZ7IftICC1rezPXXqyZGyeRZ0PJXkU5ssBOiKj+4uzvymH7D84lq3Nnmvx+iYv1zmQw6FAtZTx7ZD3NatiQOX23MZrS2JA85vHPmcu3xWSJ0Sj+QaYc6wHUzODM79cc4nWrx+qjcMiq4lVwcE2O42s2maMpaISXx2EE9sWTDTgBMOahLIO2EOxN+dJZ6jdOLpo3By0s24Qwpbr9WqCvLXWWB6mZhimU7C9GuBpW1yi0ZVQeicr0s3eDkLtEqd50FbXCjcgUNOOF+/GYPhEIq3DJO26o6gxs78wUjhZbEcpf93wX3PNKcRLJbclMN6dOEvt0bAhNXOLq2f3PuZK0taCItX9pOebxBBSr3RJiiHju4p/cWwkP9SGO5c6Us7zJ1y8jHJfE2ikqcn6s2SksxoBosEv8eGjOoJx77xnEY1Ks2LfeaVu7FTI4Oqpto9dbdeG2ZvEqgf9HifKEQabmHdRyywnDq1Neny3cRlo5WJhAKCXFQmIGrGJVyV8mmTIegaLets2DklkmikOWOmDFnQLWcoWhNuSze+v4p+PnHD/Ym94ioJukAzvXXXYZPTh1t1K7MwJ6NGD2wR2CdWBWqCUVRseN85SyeMkD1VgQU34TlhFkXHd0cKZdTr39ANsnLjXh+3C2XJP3AnkxNK3c52ZdKQbV2FPCJm2cGlLc4QUJMpqVDZTVwBabyoYfdiDrlHtei95dRXwdVfg5VR2U6YaetM2/mc9coxTDkZecYHAWkG0RLA7FmlfGn66TCDMUP7DsYZxwc/irflMvgk0eO8m37x5en4dlvf9DzVYehUuRh+k3cxePNdT53/vOK1vDdX5iKj7jnNKxvN+VC1V5bFGwzLuI91iCtghTWpkyZ5tvVBDWt3DlhA6pyGY6o4EwGCWWFuLs9j9VuKleVcg/Lphs3zl0lmmqGquhz99wyijh3lVvG1HJv7SgYTWJKEi3TKlnuC9fuADThemkhXkfVq73udb/UWPamXBafmDISf7t4KgBgeL/uGDXQGcAb0b8HPnTgXhg/pJdX/gvHjvEdf/4R/o4BcKxdnrslKG9R4PFDHMV8xuS9lW8A/A1KNNzF9NAzrzwRj379OO25kWe5u2/WCTSseAjvYHUmmDh2IHfGPHRy2j4D5cPqnppW7qql5nQEc6zE9LlLBb7wp1lY5ebj2N0efI0Oc8uIg0Uia7aqtyvXjZS+M8FyLzDm7VelH1BZ7qptqsHGtg4zyz2JW8YZCHbanLl4I97fshtvrdzSZYmbVApbPxW/NJl45kdef5M0KJ/LZnxveHIHd/n0/TH/h9Px3o8+5G0jIjx46TRle6K0zYN6YN6103HB1NHKNwAePit20B0Jksx5Y2IGZSe44ZIc8R7zfgNNRaMG9PB+D/n3Orx5AOZdOx3Hjk+2TnMtU9vKHX7LIMyilHVt3GgZWUm+sGiD9zmuW2bDDnU43aV/fT1UBhG59rbOfHFgucC8cYDALD+Nz31XW/AcVMvltRr63JP6QLnVyAefS6lL5PbPHI6vnjg+sF20aLvWcvcrd9n11JAhX6STPByQyRC6NWR9kVrZDCkXlpDplsuie2PWDTNVKHfPci/u2ysi0VcYUbfLq1edGJicJBob8spUIqdOHIqxg4tvOKp7pbsiVDktqnn2avVKZoA3YBMSLcMJs9xNpuSHKTRVtEwaeaTD2pY3XfKX171p0y2bd2Oxu6qQ7AISffMiU3/6pNGyaW0dhgOqCf3kbZ0F9GzyK880LPfBvZrQuyl4u0f53HUeoTDLnSh6nIR3KrweecZmLku+RGqi++SFyz+okSm0yWI5oWC45e7s690thwOH9w2Ui8SrOvxaDOkd7DhEQ82LllFUwycYceOsKwdWX/2fE71Vs6qRxJY7Ee1HRG8K/7YR0deJ6Boiel/YflqaAvtkcP8Wo2XMfe6dknKPehjl/aLCMXVzJEVVlUreNa67R4w66cgHO7U48fQyrZ15I597Uj85VyziIxrngQ2zpOQJSTIqha1Lo0vQ5yQnUGCylA5uBMgWd05aiUi8niP691DWFRYto+uLVOfXx01PwK+76PuPgxyqHAfx/pajZZ757+Px36fs62xzLxG/fgfunaATSsiQPt3Q12CxkkqRWLkzxhYwxiYzxiYDOAzALgD/cHdfz/cxxh5JQ1AVxZS/zvewQcwP/OJp3/eC5HOPugHlusXXYZWVnmaKYBPLXYc8iemOl5ZhpiK9AmD2tmFsuSe0oLhyFxVtnLpOmbCXcjuRei1M/4Bq8DidwiQiPPzVY7VymIQzAsU3q0ap48llyJcqwWSVJTkEUUQX5y1f22s+MsEbfOTKPfnaqf5QZZnBIQt2+KJlsv56mgf1xPihzqDwoN6O5d63ewP+dvFU3HzBYZFy7SlRk2n53E8EsJgxtjyl+oyQQyHDlM667W2Y+pMn8e85zixRbrk3uMt5ycc+NX8t9rv6P97EDVnxiQ+FciZnisr9oGseD2wzzTuiyuSnw2S9yVbTUMiET9DW3R045+aZ+NmjxYRvpm8BZ03eG8P7qX3DBFLGsYtKjysxsZzWcie9ciIqKvfpE9WdDYe/WcnRRblMBq1CBkuT1Z2yGhcPl+nZbx/vRed47UidyokHDPXcHxkqUblH7P/XZcfgT589QrlPvBX5m4RoJZ8yYShu+K/J+NqJ+3rbpo4dGGlJ//ULR+LZb6vdWvVGWsr9XAB3C98vI6I5RHQbESnzbBLRxUQ0i4hmrV+/XlUkEtkyiFI6a7a14uePOul4ufJtymVd5e4v++sZ76Gts4Albu5wuW4xw53Sci9zgK1p/VHrpIqYdASO5R5dV1LLffOudry6bBM2CjlcTP33x+83JHRAMaoersM+dthwb1uSaBlHuTszlE+aMNSozf49/UpJdiGZzLDk/UOT5hqMHtgzsEqS3BHwlZlE2UodPNbdqnv17Ybj9lVHsYguwhMOGIJrz5yIq047wNtGRDjrkOHK1B9hHD1uEEYOULu16o2SlTsRNQI4A8C97qabAIwDMBnAagC/Uh3HGLuFMTaFMTZl8OASw5QMLHfOAcOc17micudrNfqPfed9f2KqMEtc1W4MgzkRpi8GrYb5ywGzt42klvvVpx8QuW4mAGzaGUzboOooVDpXlUFS3McV2RHNxcgM0WAmheWry0AZpe+45S4uNKLiqLED8a2T98VPzvbnN5E7IpPUtd7grKvwzpq8N/4uWeoygYHcDHm/Lz/1pJZ7U0iKhSjEe7EhS/j0Uc2BFZ4s4aRhuX8IwOuMsbUAwBhbyxjLM8YKAP4AQP3elRJEYsKs6PI8VzN3QTTmMugsRFujcWLonfrLrN0NkVP+hiEPvqpo6ygYdQKyQv78sWMxvL9+5uW9lxwFANisyLqoUu6q6fvZDGkHTR3lzqMuivL73TLOX1Hh6V5AohTeLve695OUe+9uOdzwX5OL9WQIXzlxPPr1KH1pt6Jbxvn7jZP3Ra+IUD35emUyFHgWTEX556XTcPXpRev6pk8ehi8fPw77Do0/IJumW3NPJY2u8DwILhkiGsYY46tCnw3gnRTa0EIwm6HKyWUyuPOlZZ4LojGXwc62aGs0bL/qRpTXA1WhWgQ4beIod5MOSYynD0PlJw87bsro/shlCJt2BZW7yt/cmMsEcvpkI2azNoTES/PjxXJAyNhBiG4nEO754lF4bO6aQCd0wv5DcNYhwzVHirIG3TI/OfsgqTRE9AAAGphJREFUdG+MTjimclXo3kACbhmiwPUxtdwnj+yHyUKO+pEDeuA70/c3OlZGFGHlpt2xjv3S8eNwwv5DErVbT5Sk3ImoB4CTAXxR2PxzIpoMx6BeJu1LHSIKTfkr89LiDbjtxeLSe025DLbt7ogdLSOiGohsN3iNKLdfHlAnNZP5xccn4dv3zUFHZ3rRMlFhhzJEhF7dcsoQQlVz8oxOXoc2FwyKE3zyDOjdlMP2tk6fP5krQDFyRacUeTPfmb4f2jsLuOGJhcIxzpJuh43uj3mr/e49075cNXZwvpSLRkZ2K3Xko6PAAso9Szhnykg8MW8tzjt8FL5z/5ySFkdJCr/HJo/sh9MnDYt17OUJO5R6oyS3DGNsF2NsIGNsq7DtAsbYQYyxSYyxMwQrviz4Lffo8vPX+NdUddwy+jh3cdanDtU+k8HJrkhqZGK5c/9uuyJvuczmXe3488vFoKgDhvXB9z8SzNWtTCUboSOc3OmqsM/gNpV16szQNHDLFBh+98lDA2W40e+z3HWGu3syXz5+H3z9pGLExoj+3TFKGLCTrV7TDr2UiVvXnnEgmgf2wIj+3TFucC8M79fd5y4Rka9jlgiDezfhH1+e5oUZViJ0kD9Sf/3CkTWbcrfS1HT6AcB5eDyfewJt2ZjNGM1QDVOSKhdM1CIQXYXJgCr375qUXbhuB95YsQWAM4nnP187FmdODroZVMop2oqkwIzasybvrey0Vco9Q+FKkSttXboJ7n9vyAXdMkE3iZoXLj8B3YRZi7IBbjr/oRR33THjB+GZb38Q3RqcNAMvXnECpu0zSFlWPi+xU+Zeuii3zGenjcEBw9QJy5LyvY9MQFMuoxxbsZhR+8PPVLSGkmSfa3SjZaJyqatWVuJ0KHzVJoOTlWRw7yas394GoPiAxwmbBIAHL3OSVMWZ/BNGLks+d9agXo244dxDfDHvHDEWvSFL6Mg7q/TofO4Ev8+d/zqi4uIii2tmcgtdXpFJ/nVPmTAUc1q2QkZ2aZgqbdOFMUpFHrj1KXf3mYj6Kb+neHMrlfOOGIXzFJkv02Jw7yYcNkoZpV031LxyJ8B70pJYOzzOXdcv8Bt83bY25f4jxwxQJg6rFstdB39gnZWOnAdczqcuc86UEbhnVnHR7+KMSMXTn6Bva8hkfG9IPOGTquMVk0E5cjBkKEa0DCtu99cD9GjMYtKIvpjTstUbZJXfCOQJUbd8egpUyJOgWjabDQ7yUMrGbAbt+YJRKGQS5LVcRXkL3jWqvymdr111UqVFKDs175YRQyGTvMk2uT53nUuHv0av36FW7lkpex9nsyLqo5rwrXTjKr33t4Qrnm5SkqSwSS5JBotzWcJuwTU0vJ8TOqmqqrvP9cFnUobNjCW/W0ahLIuLTBTLqrI2Lv3pacaTZ2SXxpFjB2hK+tnmToIyyTp40gHJI0PktAfiGxcztNwt1UkdWO4kpPxN5pYJO5a/DWzUpOnNZkgZLXPHS8uMZdh/r96Bgd5y4y2ogKLl/u375oQeI0eo8A5C5ZNVXZOoXyeXyaBVeAsa6SbIUv023RTKPRsSLSOWK7Bih6EqLfru+anlDCY2qeBej2yG8Ob3TkaPRrNH7pCR/fDA6+9jwt598PzCDaEzVG/+1GGxXWocVaoCjpyGwFJb1IflHiPOXYYr944I5a6LfiEKDgKacniz4/PrU4HMcqKr1XR6v2y5Z4QOQibJb9GQJV/oJs9+qHqrEnW4lwMlQ9rJP0TFzsg3iUlQXNy91rtbQ8Byb0hovorXqHe3BuOcO5+aOhovXP5BTNCsrCSSy2YSz94MU+57u3l6Jo3opy1jqV5qX7kDJUXL8NF43UozvE79xBezhFtjBvUMbLvjM0fgmf8+PvZA8GUf3CdWeRVZwV8u+6l7ahY3kC33sMRSqvGPKLWWy/onJvHEXOrLU6zNs9wzFLKgdVHOfIF54XX7Di2uBcr93H27N3jXhNcm+6ZNyUpvAKYQkTa1b5qEzUc4ZFR//Odrx+KLx40tuxyW9Kl95U7FVeiTxI1zhaWzzLni1g3WZogiZ6MO79cdvz7n4MD2nk05NA/qGXusQNVRqAizEsX1LWVr9380MdEByz3D6wqWVV3OaLcM+RY+GeAm01IuM6iy3MkZlNXBdxUKDAeP7Id7vngUvnVKMUadR6j06Z7zLFr++yeNOxf9+NVI1L13wLA+iSKfLJWnDpR7MalSnPS2HO6W0VnfhQLDx256CU8vUGeuzGQoctr+1LEDQwfgROX1m3Mna8tx5GXD+MCjTNjqOeLzKncC3XJZz2UkorPcVasm8XP6/DFj8J+v6fOeizRkM75B3b7dG3116eUvyqPr0Aqs2DkNc6/XEWMGKN0SjuXu3hduxx3mvggjW9TuVUlbp9OZTh07wMvvY6kPal65b2/txO0vLsP21o5kyj0bbbnPXr5Ze3yGomPaMxSeCErsV/bfK9rHqpp6r0LObT1x7z749qn7ARDdBRRQXLksKV0t8pJiKrcM99/zN5399uodOcHlsa8f5zuWwyNFejUFxyRES5hf2wzpZ6i2dxYwqFcTfn/+obglYkGH3t0avHp4x51UuVf7YGSbG500ekBPHN5sFsljqQ1qPlqG8+6qbUbJumSi3DJR8erZTPSAajZDoSvpiD53kxA701l7snKfNKKv9wBzpdOrKRdQiBnSKHdJNpVR2pB1Fpng1raJUtxvL8fvLXeAfJGGr7kLW9/87OKijELRjGC569wn/Pc1yVPiZJcs5mcBkq8Jy10aSVU8/63j5uqJS5/udaMKLC5184sWmDpZF58EooPnnP77ayuV+3m8sQ4i0kbaiGXCLDjRn2+k3BvUSlZmYE9/Gtm+3Rt9LoyrTz8Ax+83JOCndjqjYKWyz121DFuD9CbkU+6uwj//yFH46ysrAvWLivmw0f0xcoDjPunemMWXPjDOp9xFy90LycwAOc3LqEnGywcvneYl+mqQLfeEa8ImHVDlfOkD49CRL+BTU0cnqyCCsw8djuWbduHSFAbpLdVFzbtlOAxMablf/qHwDHHcLbNBE8f+y8cWhB6fNQiFjOOWUS0FJ9PN0HIf1te/5NyI/t296fkMTo71fYb0CnR+2QwpB9ECljv/q/C588lIKotXNz4nlv3stDG+MMWAchYHVIVORqdE2w0yXh48sh/Odae8yz533v7ogfEiWLxrlNB2796YxeXT9w90rGnRlHPq72UXwqg76ke5M7VrJSrKoTFCUW7eFW65Zyg64sBRlvr9sd0ykuX+pePHYYLCr72XQrnz6yG22SwprCxRQAF/9cTxPgVz/pGjfH57zq0XHo6PHTrCcwmp3Am6txjxt9rV7s+t0r9HI86ZMsL7zkv2asp59aneNo4aOxCnHbQXDh0dL1abK/fpB+6Fkw4Yit+cewjOOHhv3Hrh4bHq4Ve5yl3vljqkzpR7UMtGTRoxHZzUYRImFhbFARSjQQ4Y1sdIHrnMwJ5NuPWiYG4TvuoU4EyYOnLMQKWSyWUz+Ogh/nVD5Zwo3zx5X1+7Pzn7IOVMzYNG9MWvzjnYm1Wq8rnrlLuYlVLO15PJEH7+8YNx0dHN+Pap+3l1/PjsA/GTsw/CwSP7YfTAHoFw2H2G9MKNnzwsdnbBz0xrxn5De+P8I0fhjxdOwcgBPfDb8w7BPkPirSrUoyGLI8cMwO/PD6YYtljKSd28ixUYS2i5l6bcZSUoMrxfd7y/ZXfo2p5A0fL/zbmTjdwyKkWlUpjD+3XH2ME9ceWHDsDJ7kLNOsUqdlLZDPmms3/9JGdAM45r4LIT9sFbLVtizW7c3lZ8S9ItLH3NGRMBAHNatmDGu2tx1LiBGNK7Gx68dJqyvOmMUJkR/XvgsW8cl+hYkUyG8Pcv2hBDS9dTN8qdQT1oFmVZJw1x44Tl9Dh14l647cWlyBKFDsgVU6uqfd0y3RQLD6t0dlNDBk9963hlOTl2PCcp943uWqa//MTB+PhhjjskzqIJU8cOxNvXnKrcp+tgNruLY//ps0doY/c5k0b0w7wfTo+UI6lyt1hqnVKX2VsGYDuAPIBOxtgUIhoA4O8AmuEss3cOY0wfKJ4SBcaUg2ZRlnupD39YFAavOpOh0JXguZ41FUW23ImKCnNAz0ZschWz6q1CN7AndioZKma6FLMvDu7dhPFDemHkgNKmxev6U55Jk6cdSAKPsBnUqwkbdrRZ5W7ZY0nD5/5Bxthkxhh3+l4B4EnG2HgAT7rfyw5jTKloox7uUtcxDYutF6M4wiJcRMvdBFXUiiopVti5y6ctpwDmyl1+S3j8G8fhtoviDSp6bbp/defJO6UhJSj3fj0asey60731Rqt9EpHFUi7KMaB6JoA73c93AjirDG0E6Myrfe5Ryj3JxCcRXc6ZA4f38VwgGQp3D/E6TBVRJkP40VkH+rbxlAQfFibpqNrkTchSZ2XL3XU3Bd8SSleWut+ETzDq36NRuT9WG0K+GYtlT6RU5c4APE5Es4noYnfbUL4otvs3+UoCMegsMGW0TJRbxmRyS1S7Mv/+yjG475KjQ7Mmini5xWMoInlws1tDFm9+72Rc85GJ3jZVu7omDhlVHPjMZYoTs8LcSUnp10Od4vhnH5uEN793ciqJqkocSrFYap5SH4FpjLFDAXwIwKVEZBxeQEQXE9EsIpq1fr06KVccOvIFpeUepVhLXQ6Pdw48ZwvgKK9uDVlPaUcN2nJXShz/8Mj+wQHHfj0afYtKqHzuPHf8ISP9USziItdiSoVSQ0VVDOipdrs0ZDPol4LVDghZL1OpzWKpPUp6chljq9y/6wD8A8ARANYS0TAAcP+u0xx7C2NsCmNsyuDBg0sRA4DjXlG5WKJygpjkYo9qF/DnBS8u5+YoSFW45cGCcuUTkGRr/NjxwRXreTKtI8cO9KJXdK4SVYDO0D7d8PBXj8GPzj4wsI+nK3AGVJlSpjTor7Hc04TL3aNMMzstlmoncbQMEfUEkGGMbXc/nwLgWgAPAbgQwHXu3wfTEDSKzoLacs9G5ASRk2vFhfvLRQXOXUGd3kQev/K9/0tH+ybD/Oa8QzBv9TYMkHLB7DOkF55fuMH7PnXsAPz2vEO87weP6Isn5yv7TgD6GPyJe6tTAReXlCMvJUEcy10VoikyckAPzGnZir0jwhzT4OOHObNkS1lf1GKpZUoJhRwK4B+u1ZgD8FfG2KNE9BqAe4jocwBWAPhE6WJG05FnygRhYZOMAOBDB+5VUrsqBZ6LyAV+2Gh/rvReTTllulX5TeSAYX0wpHdx1unAXk5n0EOzclLSMMAMiW4ZM8v37i9Mjcy78rOPTcJHJu0dmQI4Dfp2b/Di8y2WPZHEyp0xtgRAYHkhxthGACeWIlQSOvMFpVsmSsEREaaOHYCXl2xK1C633EULV84omDRdrOwykheK/v5HJuLgkf1w9LiByuPjR7YUc7TwpqKscc5RGhlEejXlMN3tTG+9cAo+d+esmPJZLBZT6iamwImWiR8KaUJYHVwBN2aLFi7PAMk7m7hLtPFokrwUySOvEduzKYdPHjk6lfBEEVHeuDlZTDnxAHV6AYvFkg51o9w78gw72zoD202Ue9Q8pjDXTl5hnTdIqxGFpftVwafe80Ra/BwSLDSVCDEUsdTcOxaLpTLUTW6ZG59Z5C1wLBLXct9/r96Yv2Y7AGdQ7r7ZLU4deXV5lc+dW9Le4sruvps/dRjWbmuNlGFE/+6Yu2obWjbvAuC4fHa1531pessB78PEzqyc0/f/74LDsFpYM9VisaRH3Zhl21s7lRZ41IAq4J889JtznWiU8UN64dSJjn84zK3CrXNVVE5xn3P89AP3woVHN0fKc/XpE/DB/Qbjt+cdgqPGDsQlHxjnqy+KP356ii/3uSnewhIE3HbRlLIPSJ46cS9cNG1MWduwWPZU6sZy12HijhY7Be6GaOsseJZy2ICo6Ff/xccn4YHX3/f28TGAuD73kQN64PbPHAEAuPviqbh3lrMEoGlI/kkThmpT5prAGHDC/kNxwv7WL26x1Cp1Y7mnwV2fPxLD+nZD94YsvjN9P292I7fK99+rN34sTf7JC66XT0wZibsvnhrYFxVrHwW3/EtNcmaxWPYc6l65x8kKSOTMbJz3w+n48KS9ve05KakW5/PHjMGlJzgLC6uSXck+96RkpUlR5eIrJzqLcuhyv1gsltphj3bL8EFQb51LKa0WTwvw0UOH48ZnFrtlHM6ZMgJXf3gCAOACzcr0xWiZ0pQ771DKbblfMHW09lwsFkttUfeWu6jcv3z8OCy77nT8012SLSqh18gBPbDsutN9VjxXtCZ6lvvcS4048ZR7mS13i8VSP9S/ckfQpcIjaLhFHaV6vRzorFjYRM92Joxzl+F9kPW5WywWU+peuTMh6Wtx2TvnL7fco1SmaP17lrtBMtm0fO4j+js5W+IsNm2xWPZs6t7n7oNb7q6Wj6t0GRj27uck7ho3uFdEacdn/9bKLV4q3aQcOLwvZnzjOKM2LRaLBdgDlHvPxuIpcsudu2Vkn7tJYM3R4wbh3kuOwqGj+keWveaMCThnygiMHtjTXGAN44V88RaLxRJF3btlRg7o4eX05hY0aZS7zqUtR9Ec3jzAaJC0KZfFIQadgMVisaRN3VvuAPB/F0zBA6+34KOHOtPp5RBFrvSj1gu145kWi6VWqHnL/a7PH4nzjxwVWiabcWaPcmvbSwvgWu7XfXQSrj1zYmBdUfF4wEmxa7FYLLVAzSv3afsMwhGKVYzC4JY7X8Gob48GfPqoZm1e9HGDe+Ly6fvjpk8dWpqwFovF0kXUhSmaiTlJ6KDhffGFY8cYZyQkInzp+HFJRLNYLJaKkNhyJ6KRRPQ0Ec0jorlE9DV3+zVE9D4Rven+Oy09cdXEnd6fyRCuOn2CtyiGxWKx1BulWO6dAL7FGHudiHoDmE1EM9x91zPGflm6eGaUc0EJi8ViqUVKWSB7NYDV7uftRDQPwPC0BIuDyYIcFovFsieRyoAqETUDOATAK+6my4hoDhHdRkTKQG8iupiIZhHRrPXr15fUfrbE6f0Wi8VSb5Ss3ImoF4D7AXydMbYNwE0AxgGYDMey/5XqOMbYLYyxKYyxKYMHDy5JhlJT6losFku9UZJyJ6IGOIr9LsbYAwDAGFvLGMszxgoA/gDgiNLFDMe6ZSwWi8VPKdEyBOBWAPMYY78Wtg8Tip0N4J3k4plhB1QtFovFTynRMtMAXADgbSJ60932PwDOI6LJcDLpLgPwxZIkNKDUlLoWi8VSb5QSLfMC1OtcPJJcnGTwHOvdG7LY3ZHv6uYtFoul6qj59ANAcaUjnk7AYrFY9nTqQrlzn3t3q9wtFosFQJ0odx4s073BKneLxWIB6kS5t7p+dtktY900FotlT6UuskLyhaj79mjEPy+dhrXbWrF1VweOGBMvFbDFYrHUC3Wh3A8b1R9fPn4cPjNtDAb3bqq0OBaLxVJx6kK5ZzKE70zfv9JiWCwWS9VQFz53i8Visfixyt1isVjqEKvcLRaLpQ6xyt1isVjqEKvcLRaLpQ6xyt1isVjqEKvcLRaLpQ6xyt1isVjqEGKMVVoGENF6AMtLqGIQgA0piVMurIzpUQty1oKMQG3IWQsyApWRczRjTLkIdVUo91IholmMsSmVliMMK2N61IKctSAjUBty1oKMQPXJad0yFovFUodY5W6xWCx1SL0o91sqLYABVsb0qAU5a0FGoDbkrAUZgSqTsy587haLxWLxUy+Wu8VisVgErHK3WCyWOqSmlTsRTSeiBUS0iIiuqLAstxHROiJ6R9g2gIhmENFC929/Yd+VrtwLiOjULpJxJBE9TUTziGguEX2t2uQkom5E9CoRveXK+INqk1FoN0tEbxDRv6tYxmVE9DYRvUlEs6pYzn5EdB8RzXfvz6OqSU4i2s+9hvzfNiL6ejXJGIAxVpP/AGQBLAYwFkAjgLcATKigPMcBOBTAO8K2nwO4wv18BYCfuZ8nuPI2ARjjnke2C2QcBuBQ93NvAO+5slSNnAAIQC/3cwOAVwBMrSYZBVm/CeCvAP5djb+32/YyAIOkbdUo550APu9+bgTQrxrldNvPAlgDYHS1ysgYq2nlfhSAx4TvVwK4ssIyNcOv3BcAGOZ+HgZggUpWAI8BOKoC8j4I4ORqlRNADwCvAziy2mQEMALAkwBOEJR7VcnotqVS7lUlJ4A+AJbCDfCoVjmF9k4B8GI1y8gYq2m3zHAAK4XvLe62amIoY2w1ALh/h7jbKy47ETUDOASOZVxVcrrujjcBrAMwgzFWdTICuAHAdwAUhG3VJiMAMACPE9FsIrq4SuUcC2A9gNtdN9cfiahnFcrJORfA3e7napWxppU7KbbVSlxnRWUnol4A7gfwdcbYtrCiim1ll5MxlmeMTYZjHR9BRAeGFO9yGYnowwDWMcZmmx6i2NZVv/c0xtihAD4E4FIiOi6kbKXkzMFxad7EGDsEwE44Lg4dFbueRNQI4AwA90YVVWzrUv1Uy8q9BcBI4fsIAKsqJIuOtUQ0DADcv+vc7RWTnYga4Cj2uxhjD1SrnADAGNsC4BkA06tMxmkAziCiZQD+BuAEIvpLlckIAGCMrXL/rgPwDwBHVKGcLQBa3Dc0ALgPjrKvNjkBp5N8nTG21v1ejTICqG3l/hqA8UQ0xu1NzwXwUIVlknkIwIXu5wvh+Lj59nOJqImIxgAYD+DVcgtDRATgVgDzGGO/rkY5iWgwEfVzP3cHcBKA+dUkI2PsSsbYCMZYM5z77inG2KeqSUYAIKKeRNSbf4bjK36n2uRkjK0BsJKI9nM3nQjg3WqT0+U8FF0yXJZqk9GhKx38ZRjYOA1OxMdiAFdVWJa7AawG0AGn1/4cgIFwBt0Wun8HCOWvcuVeAOBDXSTjMXBeDecAeNP9d1o1yQlgEoA3XBnfAfA9d3vVyCjJezyKA6pVJSMcX/Zb7r+5/BmpNjnddicDmOX+7v8E0L/a5IQzwL8RQF9hW1XJKP6z6QcsFoulDqllt4zFYrFYNFjlbrFYLHWIVe4Wi8VSh1jlbrFYLHWIVe4Wi8VSh1jlbrFYLHWIVe4Wi8VSh/w/YmzUfsIuAf8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > 195:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
